aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorradix <radix@yandex-team.ru>2022-02-10 16:49:44 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:49:44 +0300
commitbfd0aa05cabedcab6f4c4d36948dc7581e476782 (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parent24658cfd172ef5de124df383c334079783154fbf (diff)
downloadydb-bfd0aa05cabedcab6f4c4d36948dc7581e476782.tar.gz
Restoring authorship annotation for <radix@yandex-team.ru>. Commit 2 of 2.
-rw-r--r--library/cpp/http/misc/parsed_request.cpp4
-rw-r--r--util/folder/pathsplit.cpp2
-rw-r--r--util/generic/strbuf.h94
-rw-r--r--ydb/core/base/appdata.h10
-rw-r--r--ydb/core/base/events.h4
-rw-r--r--ydb/core/cms/console/console.cpp30
-rw-r--r--ydb/core/cms/console/console_impl.h6
-rw-r--r--ydb/core/cms/console/net_classifier_updater.cpp512
-rw-r--r--ydb/core/cms/console/net_classifier_updater.h22
-rw-r--r--ydb/core/cms/console/net_classifier_updater_ut.cpp234
-rw-r--r--ydb/core/cms/console/ut/ya.make2
-rw-r--r--ydb/core/cms/console/ya.make2
-rw-r--r--ydb/core/driver_lib/cli_utils/cli.h2
-rw-r--r--ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp4
-rw-r--r--ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp8
-rw-r--r--ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp266
-rw-r--r--ydb/core/driver_lib/cli_utils/ya.make2
-rw-r--r--ydb/core/driver_lib/run/config.h6
-rw-r--r--ydb/core/driver_lib/run/config_parser.cpp20
-rw-r--r--ydb/core/driver_lib/run/driver.h2
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp56
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.h20
-rw-r--r--ydb/core/driver_lib/run/main.cpp4
-rw-r--r--ydb/core/driver_lib/run/run.cpp76
-rw-r--r--ydb/core/grpc_services/base/base.h2
-rw-r--r--ydb/core/grpc_services/grpc_request_proxy.h2
-rw-r--r--ydb/core/mind/address_classification/counters.cpp38
-rw-r--r--ydb/core/mind/address_classification/counters.h50
-rw-r--r--ydb/core/mind/address_classification/net_classifier.cpp812
-rw-r--r--ydb/core/mind/address_classification/net_classifier.h92
-rw-r--r--ydb/core/mind/address_classification/net_classifier_ut.cpp396
-rw-r--r--ydb/core/mind/address_classification/ut/ya.make44
-rw-r--r--ydb/core/mind/address_classification/ya.make36
-rw-r--r--ydb/core/persqueue/cluster_tracker.cpp312
-rw-r--r--ydb/core/persqueue/cluster_tracker.h142
-rw-r--r--ydb/core/persqueue/ya.make10
-rw-r--r--ydb/core/protos/config.proto62
-rw-r--r--ydb/core/protos/console_config.proto8
-rw-r--r--ydb/core/protos/msgbus.proto22
-rw-r--r--ydb/core/protos/netclassifier.proto76
-rw-r--r--ydb/core/protos/pqconfig.proto28
-rw-r--r--ydb/core/protos/services.proto16
-rw-r--r--ydb/core/protos/sqs.proto236
-rw-r--r--ydb/core/protos/ya.make2
-rw-r--r--ydb/core/testlib/actors/test_runtime.cpp2
-rw-r--r--ydb/core/testlib/basics/appdata.h2
-rw-r--r--ydb/core/testlib/test_client.cpp12
-rw-r--r--ydb/core/testlib/test_client.h14
-rw-r--r--ydb/core/testlib/test_pq_client.h44
-rw-r--r--ydb/core/util/address_classifier.cpp190
-rw-r--r--ydb/core/util/address_classifier.h148
-rw-r--r--ydb/core/util/address_classifier_ut.cpp266
-rw-r--r--ydb/core/util/ut/ya.make2
-rw-r--r--ydb/core/util/ya.make2
-rw-r--r--ydb/core/ymq/actor/action.h442
-rw-r--r--ydb/core/ymq/actor/actor.cpp8
-rw-r--r--ydb/core/ymq/actor/count_queues.cpp120
-rw-r--r--ydb/core/ymq/actor/create_queue.cpp82
-rw-r--r--ydb/core/ymq/actor/create_user.cpp2
-rw-r--r--ydb/core/ymq/actor/delete_message.cpp22
-rw-r--r--ydb/core/ymq/actor/delete_queue.cpp16
-rw-r--r--ydb/core/ymq/actor/delete_user.cpp10
-rw-r--r--ydb/core/ymq/actor/error.cpp6
-rw-r--r--ydb/core/ymq/actor/events.h282
-rw-r--r--ydb/core/ymq/actor/executor.cpp30
-rw-r--r--ydb/core/ymq/actor/executor.h28
-rw-r--r--ydb/core/ymq/actor/garbage_collector.cpp1538
-rw-r--r--ydb/core/ymq/actor/garbage_collector.h14
-rw-r--r--ydb/core/ymq/actor/get_queue_attributes.cpp134
-rw-r--r--ydb/core/ymq/actor/get_queue_url.cpp16
-rw-r--r--ydb/core/ymq/actor/infly.cpp4
-rw-r--r--ydb/core/ymq/actor/infly.h28
-rw-r--r--ydb/core/ymq/actor/list_dead_letter_source_queues.cpp182
-rw-r--r--ydb/core/ymq/actor/list_permissions.cpp260
-rw-r--r--ydb/core/ymq/actor/list_queues.cpp32
-rw-r--r--ydb/core/ymq/actor/list_users.cpp2
-rw-r--r--ydb/core/ymq/actor/metering.cpp384
-rw-r--r--ydb/core/ymq/actor/metering.h124
-rw-r--r--ydb/core/ymq/actor/migration.cpp2
-rw-r--r--ydb/core/ymq/actor/modify_permissions.cpp268
-rw-r--r--ydb/core/ymq/actor/params.h34
-rw-r--r--ydb/core/ymq/actor/proxy_actor.cpp68
-rw-r--r--ydb/core/ymq/actor/proxy_actor.h38
-rw-r--r--ydb/core/ymq/actor/purge_queue.cpp8
-rw-r--r--ydb/core/ymq/actor/queue_leader.cpp360
-rw-r--r--ydb/core/ymq/actor/queue_leader.h38
-rw-r--r--ydb/core/ymq/actor/queue_schema.cpp1658
-rw-r--r--ydb/core/ymq/actor/queue_schema.h244
-rw-r--r--ydb/core/ymq/actor/queues_list_reader.cpp54
-rw-r--r--ydb/core/ymq/actor/queues_list_reader.h4
-rw-r--r--ydb/core/ymq/actor/receive_message.cpp6
-rw-r--r--ydb/core/ymq/actor/schema.cpp182
-rw-r--r--ydb/core/ymq/actor/schema.h70
-rw-r--r--ydb/core/ymq/actor/service.cpp146
-rw-r--r--ydb/core/ymq/actor/service.h16
-rw-r--r--ydb/core/ymq/actor/serviceid.h14
-rw-r--r--ydb/core/ymq/actor/set_queue_attributes.cpp76
-rw-r--r--ydb/core/ymq/actor/ut/infly_ut.cpp30
-rw-r--r--ydb/core/ymq/actor/ya.make18
-rw-r--r--ydb/core/ymq/base/acl.cpp274
-rw-r--r--ydb/core/ymq/base/acl.h32
-rw-r--r--ydb/core/ymq/base/action.h8
-rw-r--r--ydb/core/ymq/base/cloud_enums.h32
-rw-r--r--ydb/core/ymq/base/constants.h6
-rw-r--r--ydb/core/ymq/base/counters.cpp110
-rw-r--r--ydb/core/ymq/base/counters.h34
-rw-r--r--ydb/core/ymq/base/dlq_helpers.cpp194
-rw-r--r--ydb/core/ymq/base/dlq_helpers.h44
-rw-r--r--ydb/core/ymq/base/limits.h10
-rw-r--r--ydb/core/ymq/base/processed_request_attributes.h50
-rw-r--r--ydb/core/ymq/base/query_id.h6
-rw-r--r--ydb/core/ymq/base/queue_attributes.cpp218
-rw-r--r--ydb/core/ymq/base/queue_attributes.h64
-rw-r--r--ydb/core/ymq/base/queue_id.cpp144
-rw-r--r--ydb/core/ymq/base/queue_id.h14
-rw-r--r--ydb/core/ymq/base/queue_path.h28
-rw-r--r--ydb/core/ymq/base/security.h56
-rw-r--r--ydb/core/ymq/base/ut/dlq_helpers_ut.cpp116
-rw-r--r--ydb/core/ymq/base/ut/queue_attributes_ut.cpp92
-rw-r--r--ydb/core/ymq/base/ut/ya.make4
-rw-r--r--ydb/core/ymq/base/ya.make12
-rw-r--r--ydb/core/ymq/client/bin/main.cpp192
-rw-r--r--ydb/core/ymq/client/cpp/client.cpp20
-rw-r--r--ydb/core/ymq/client/cpp/client.h8
-rw-r--r--ydb/core/ymq/http/http.cpp344
-rw-r--r--ydb/core/ymq/http/http.h18
-rw-r--r--ydb/core/ymq/http/params.h8
-rw-r--r--ydb/core/ymq/http/parser.rl618
-rw-r--r--ydb/core/ymq/http/types.h6
-rw-r--r--ydb/core/ymq/http/xml.cpp220
-rw-r--r--ydb/core/ymq/queues/fifo/queries.cpp698
-rw-r--r--ydb/core/ymq/queues/fifo/schema.cpp8
-rw-r--r--ydb/core/ymq/queues/std/queries.cpp596
-rw-r--r--ydb/core/ymq/queues/std/schema.cpp10
-rw-r--r--ydb/core/ymq/ut/queue_id_ut.cpp52
-rw-r--r--ydb/core/ymq/ut/ya.make2
-rw-r--r--ydb/library/aclib/aclib.cpp12
-rw-r--r--ydb/library/aclib/aclib_ut.cpp90
-rw-r--r--ydb/library/http_proxy/authorization/auth_helpers.cpp2
-rw-r--r--ydb/library/http_proxy/authorization/signature.cpp498
-rw-r--r--ydb/library/http_proxy/authorization/signature.h100
-rw-r--r--ydb/library/http_proxy/authorization/ut/signature_ut.cpp218
-rw-r--r--ydb/library/http_proxy/error/error.cpp12
-rw-r--r--ydb/library/http_proxy/error/error.h2
-rw-r--r--ydb/public/api/grpc/draft/ydb_persqueue_v1.proto12
-rw-r--r--ydb/public/api/protos/ya.make4
-rw-r--r--ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto168
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp550
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h24
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp1326
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp568
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h42
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make26
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp2
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h96
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp104
-rw-r--r--ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make22
-rw-r--r--ydb/services/persqueue_cluster_discovery/counters.cpp112
-rw-r--r--ydb/services/persqueue_cluster_discovery/counters.h98
-rw-r--r--ydb/services/persqueue_cluster_discovery/grpc_service.cpp114
-rw-r--r--ydb/services/persqueue_cluster_discovery/grpc_service.h40
-rw-r--r--ydb/services/persqueue_cluster_discovery/ut/ya.make40
-rw-r--r--ydb/services/persqueue_cluster_discovery/ya.make34
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_read.cpp40
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_read.h4
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_write.cpp22
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_write.h4
-rw-r--r--ydb/tests/functional/sqs/sqs_requests_client.py148
-rw-r--r--ydb/tests/functional/sqs/sqs_test_base.py108
-rw-r--r--ydb/tests/functional/sqs/test_account_actions.py44
-rw-r--r--ydb/tests/functional/sqs/test_acl.py306
-rw-r--r--ydb/tests/functional/sqs/test_fifo_messaging.py56
-rw-r--r--ydb/tests/functional/sqs/test_garbage_collection.py2
-rw-r--r--ydb/tests/functional/sqs/test_generic_messaging.py30
-rw-r--r--ydb/tests/functional/sqs/test_multinode_cluster.py2
-rw-r--r--ydb/tests/functional/sqs/test_polling.py56
-rw-r--r--ydb/tests/functional/sqs/test_queue_attributes_validation.py336
-rw-r--r--ydb/tests/functional/sqs/test_queues_managing.py24
-rw-r--r--ydb/tests/functional/sqs/test_recompiles_requests.py2
-rw-r--r--ydb/tests/functional/sqs/ya.make4
180 files changed, 10404 insertions, 10404 deletions
diff --git a/library/cpp/http/misc/parsed_request.cpp b/library/cpp/http/misc/parsed_request.cpp
index baef4221a7..e332a24e91 100644
--- a/library/cpp/http/misc/parsed_request.cpp
+++ b/library/cpp/http/misc/parsed_request.cpp
@@ -16,11 +16,11 @@ static inline TStringBuf StripLeft(const TStringBuf& s) noexcept {
TParsedHttpRequest::TParsedHttpRequest(const TStringBuf& str) {
TStringBuf tmp;
- if (!StripLeft(str).TrySplit(' ', Method, tmp)) {
+ if (!StripLeft(str).TrySplit(' ', Method, tmp)) {
ythrow yexception() << "bad request(" << ToString(str).Quote() << ")";
}
- if (!StripLeft(tmp).TrySplit(' ', Request, Proto)) {
+ if (!StripLeft(tmp).TrySplit(' ', Request, Proto)) {
ythrow yexception() << "bad request(" << ToString(str).Quote() << ")";
}
diff --git a/util/folder/pathsplit.cpp b/util/folder/pathsplit.cpp
index 0477fbc033..81d439a727 100644
--- a/util/folder/pathsplit.cpp
+++ b/util/folder/pathsplit.cpp
@@ -36,7 +36,7 @@ void TPathSplitTraitsUnix::DoParsePart(const TStringBuf part0) {
TStringBuf next(part0);
TStringBuf part;
- while (TStringBuf(next).TrySplit('/', part, next)) {
+ while (TStringBuf(next).TrySplit('/', part, next)) {
AppendComponent(part);
}
diff --git a/util/generic/strbuf.h b/util/generic/strbuf.h
index e23d257fd4..70b9360d58 100644
--- a/util/generic/strbuf.h
+++ b/util/generic/strbuf.h
@@ -171,63 +171,63 @@ public:
return data() != nullptr;
}
-public:
- /**
- * Tries to split string in two parts using given delimiter character.
- * Searches for the delimiter, scanning string from the beginning.
- * The delimiter is excluded from the result. Both out parameters are
- * left unmodified if there was no delimiter character in string.
- *
- * @param[in] delim Delimiter character.
- * @param[out] l The first part of split result.
- * @param[out] r The second part of split result.
- * @returns Whether the split was actually performed.
- */
+public:
+ /**
+ * Tries to split string in two parts using given delimiter character.
+ * Searches for the delimiter, scanning string from the beginning.
+ * The delimiter is excluded from the result. Both out parameters are
+ * left unmodified if there was no delimiter character in string.
+ *
+ * @param[in] delim Delimiter character.
+ * @param[out] l The first part of split result.
+ * @param[out] r The second part of split result.
+ * @returns Whether the split was actually performed.
+ */
inline bool TrySplit(TCharType delim, TdSelf& l, TdSelf& r) const noexcept {
return TrySplitOn(TBase::find(delim), l, r);
}
- /**
- * Tries to split string in two parts using given delimiter character.
- * Searches for the delimiter, scanning string from the end.
- * The delimiter is excluded from the result. Both out parameters are
- * left unmodified if there was no delimiter character in string.
- *
- * @param[in] delim Delimiter character.
- * @param[out] l The first part of split result.
- * @param[out] r The second part of split result.
- * @returns Whether the split was actually performed.
- */
+ /**
+ * Tries to split string in two parts using given delimiter character.
+ * Searches for the delimiter, scanning string from the end.
+ * The delimiter is excluded from the result. Both out parameters are
+ * left unmodified if there was no delimiter character in string.
+ *
+ * @param[in] delim Delimiter character.
+ * @param[out] l The first part of split result.
+ * @param[out] r The second part of split result.
+ * @returns Whether the split was actually performed.
+ */
inline bool TryRSplit(TCharType delim, TdSelf& l, TdSelf& r) const noexcept {
return TrySplitOn(TBase::rfind(delim), l, r);
}
- /**
- * Tries to split string in two parts using given delimiter sequence.
- * Searches for the delimiter, scanning string from the beginning.
- * The delimiter sequence is excluded from the result. Both out parameters
- * are left unmodified if there was no delimiter character in string.
- *
- * @param[in] delim Delimiter sequence.
- * @param[out] l The first part of split result.
- * @param[out] r The second part of split result.
- * @returns Whether the split was actually performed.
- */
+ /**
+ * Tries to split string in two parts using given delimiter sequence.
+ * Searches for the delimiter, scanning string from the beginning.
+ * The delimiter sequence is excluded from the result. Both out parameters
+ * are left unmodified if there was no delimiter character in string.
+ *
+ * @param[in] delim Delimiter sequence.
+ * @param[out] l The first part of split result.
+ * @param[out] r The second part of split result.
+ * @returns Whether the split was actually performed.
+ */
inline bool TrySplit(TdSelf delim, TdSelf& l, TdSelf& r) const noexcept {
return TrySplitOn(TBase::find(delim), l, r, delim.size());
}
- /**
- * Tries to split string in two parts using given delimiter sequence.
- * Searches for the delimiter, scanning string from the end.
- * The delimiter sequence is excluded from the result. Both out parameters
- * are left unmodified if there was no delimiter character in string.
- *
- * @param[in] delim Delimiter sequence.
- * @param[out] l The first part of split result.
- * @param[out] r The second part of split result.
- * @returns Whether the split was actually performed.
- */
+ /**
+ * Tries to split string in two parts using given delimiter sequence.
+ * Searches for the delimiter, scanning string from the end.
+ * The delimiter sequence is excluded from the result. Both out parameters
+ * are left unmodified if there was no delimiter character in string.
+ *
+ * @param[in] delim Delimiter sequence.
+ * @param[out] l The first part of split result.
+ * @param[out] r The second part of split result.
+ * @returns Whether the split was actually performed.
+ */
inline bool TryRSplit(TdSelf delim, TdSelf& l, TdSelf& r) const noexcept {
return TrySplitOn(TBase::rfind(delim), l, r, delim.size());
}
@@ -521,7 +521,7 @@ private:
template <typename TDelimiterType>
inline void SplitTemplate(TDelimiterType delim, TdSelf& l, TdSelf& r) const noexcept {
- if (!TrySplit(delim, l, r)) {
+ if (!TrySplit(delim, l, r)) {
l = *this;
r = TdSelf();
}
@@ -529,7 +529,7 @@ private:
template <typename TDelimiterType>
inline void RSplitTemplate(TDelimiterType delim, TdSelf& l, TdSelf& r) const noexcept {
- if (!TryRSplit(delim, l, r)) {
+ if (!TryRSplit(delim, l, r)) {
r = *this;
l = TdSelf();
}
diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h
index b70bda343c..c666f7468c 100644
--- a/ydb/core/base/appdata.h
+++ b/ydb/core/base/appdata.h
@@ -131,13 +131,13 @@ struct TAppData {
THolder<NKikimrBlobStorage::TNodeWardenServiceSet> StaticBlobStorageConfig;
THolder<NKikimrCms::TCmsConfig> DefaultCmsConfig;
-
+
NKikimrStream::TStreamingConfig StreamingConfig;
NKikimrPQ::TPQConfig PQConfig;
- NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig;
- NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig;
- NKikimrNetClassifier::TNetClassifierDistributableConfig NetClassifierDistributableConfig;
- NKikimrConfig::TSqsConfig SqsConfig;
+ NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig;
+ NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig;
+ NKikimrNetClassifier::TNetClassifierDistributableConfig NetClassifierDistributableConfig;
+ NKikimrConfig::TSqsConfig SqsConfig;
NKikimrProto::TAuthConfig AuthConfig;
NKikimrProto::TKeyConfig KeyConfig;
NKikimrProto::TKeyConfig PDiskKeyConfig;
diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h
index 8ed9dc4396..f5fedfe19b 100644
--- a/ydb/core/base/events.h
+++ b/ydb/core/base/events.h
@@ -109,8 +109,8 @@ struct TKikimrEvents : TEvents {
ES_GRPC_REQUEST_PROXY,
ES_EXPORT_SERVICE,
ES_TX_ALLOCATOR_CLIENT,
- ES_PQ_CLUSTER_TRACKER,
- ES_NET_CLASSIFIER,
+ ES_PQ_CLUSTER_TRACKER,
+ ES_NET_CLASSIFIER,
ES_SYSTEM_VIEW,
ES_TENANT_NODE_ENUMERATOR,
ES_SERVICE_ACCOUNT_SERVICE,
diff --git a/ydb/core/cms/console/console.cpp b/ydb/core/cms/console/console.cpp
index 7747a1f157..fe477202af 100644
--- a/ydb/core/cms/console/console.cpp
+++ b/ydb/core/cms/console/console.cpp
@@ -3,8 +3,8 @@
#include "console_tenants_manager.h"
#include "http.h"
-#include "net_classifier_updater.h"
-
+#include "net_classifier_updater.h"
+
#include <ydb/core/base/counters.h>
#include <ydb/core/cms/console/validators/registry.h>
@@ -33,10 +33,10 @@ void TConsole::OnActivateExecutor(const TActorContext &ctx)
AppData()->FeatureFlags);
ctx.RegisterWithSameMailbox(TenantsManager);
- if (AppData(ctx)->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()) {
- NetClassifierUpdaterId = ctx.Register(NNetClassifierUpdater::MakeNetClassifierUpdaterActor(SelfId()));
- }
-
+ if (AppData(ctx)->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl()) {
+ NetClassifierUpdaterId = ctx.Register(NNetClassifierUpdater::MakeNetClassifierUpdaterActor(SelfId()));
+ }
+
TxProcessor->ProcessTx(CreateTxInitScheme(), ctx);
}
@@ -109,11 +109,11 @@ void TConsole::Cleanup(const TActorContext &ctx)
TenantsManager = nullptr;
}
- if (NetClassifierUpdaterId) {
- Send(NetClassifierUpdaterId, new TEvents::TEvPoisonPill);
- NetClassifierUpdaterId = {};
- }
-
+ if (NetClassifierUpdaterId) {
+ Send(NetClassifierUpdaterId, new TEvents::TEvPoisonPill);
+ NetClassifierUpdaterId = {};
+ }
+
TxProcessor->Clear();
}
@@ -145,12 +145,12 @@ void TConsole::ProcessEnqueuedEvents(const TActorContext &ctx)
void TConsole::ClearState()
{
Config.Clear();
- if (ConfigsManager) {
+ if (ConfigsManager) {
ConfigsManager->ClearState();
- }
- if (TenantsManager) {
+ }
+ if (TenantsManager) {
TenantsManager->ClearState();
- }
+ }
Counters->ResetCounters();
}
diff --git a/ydb/core/cms/console/console_impl.h b/ydb/core/cms/console/console_impl.h
index a42ee8ee05..0894337c3e 100644
--- a/ydb/core/cms/console/console_impl.h
+++ b/ydb/core/cms/console/console_impl.h
@@ -162,9 +162,9 @@ private:
TTxProcessor::TPtr TxProcessor;
TDynamicCounterPtr Counters;
- TConfigsManager* ConfigsManager;
- TTenantsManager* TenantsManager;
-
+ TConfigsManager* ConfigsManager;
+ TTenantsManager* TenantsManager;
+
TActorId NetClassifierUpdaterId;
};
diff --git a/ydb/core/cms/console/net_classifier_updater.cpp b/ydb/core/cms/console/net_classifier_updater.cpp
index 4c25545f88..121a65efc3 100644
--- a/ydb/core/cms/console/net_classifier_updater.cpp
+++ b/ydb/core/cms/console/net_classifier_updater.cpp
@@ -1,15 +1,15 @@
-#include "net_classifier_updater.h"
-
+#include "net_classifier_updater.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/tablet_pipe.h>
-
+
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/http/http_proxy.h>
#include <library/cpp/actors/interconnect/interconnect.h>
#include <library/cpp/json/json_reader.h>
-
-#include <util/stream/zlib.h>
-
+
+#include <util/stream/zlib.h>
+
#if defined BLOG_D || defined BLOG_I || defined BLOG_ERROR || defined BLOG_NOTICE
#error log macro definition clash
#endif
@@ -20,159 +20,159 @@
#define BLOG_ERROR(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::CMS_CONFIGS, stream)
-namespace NKikimr::NNetClassifierUpdater {
-
-using namespace NConsole;
-
-// helps to find a particular config item. DO NOT CHANGE this constant
-static const TString COOKIE = "NetClassifierPackedNetDataFromExternalSource";
-
-// prevents duplicates in config list. DO NOT CHANGE this constant
-static constexpr ui32 CONFIG_ORDER = 10;
-
-static TString PackNetData(const TString& netData, size_t level = 6) {
- TString result;
- result.reserve(std::size(netData));
- TStringOutput stream(result);
- TZLibCompress(&stream, ZLib::GZip, level).Write(netData);
- return result;
-}
-
-/*
- ***NetClassifier Updater algorithm***
-
- The main idea is to keep only one special config item to periodically modify it.
- All discovery nodes are subscribed to config dispatcher to get updates.
-
- Initing stage (requires WakeUp event to start):
- 1) Look for the config item via cookie
- if has distributable config item:
- GOTO Working stage
- else:
- Add distributable empty config item with cookie and order
- GOTO Working stage if added successfully
- Working stage (also requires WakeUp event to start):
- 1) Get Networks Data from an external http(s) resource
- 2) Find current config id and generation through GetConfigItem
- 3) Send ModifyEvent using step 2 information.
- 4) Schedule WakeUp event on success
-
- In case of any error algorithm falls back to Initing stage and schedules WakeUp.
-*/
-
-class NetClassifierUpdater : public TActorBootstrapped<NetClassifierUpdater> {
-private:
- using TBase = TActorBootstrapped<NetClassifierUpdater>;
-
-public:
+namespace NKikimr::NNetClassifierUpdater {
+
+using namespace NConsole;
+
+// helps to find a particular config item. DO NOT CHANGE this constant
+static const TString COOKIE = "NetClassifierPackedNetDataFromExternalSource";
+
+// prevents duplicates in config list. DO NOT CHANGE this constant
+static constexpr ui32 CONFIG_ORDER = 10;
+
+static TString PackNetData(const TString& netData, size_t level = 6) {
+ TString result;
+ result.reserve(std::size(netData));
+ TStringOutput stream(result);
+ TZLibCompress(&stream, ZLib::GZip, level).Write(netData);
+ return result;
+}
+
+/*
+ ***NetClassifier Updater algorithm***
+
+ The main idea is to keep only one special config item to periodically modify it.
+ All discovery nodes are subscribed to config dispatcher to get updates.
+
+ Initing stage (requires WakeUp event to start):
+ 1) Look for the config item via cookie
+ if has distributable config item:
+ GOTO Working stage
+ else:
+ Add distributable empty config item with cookie and order
+ GOTO Working stage if added successfully
+ Working stage (also requires WakeUp event to start):
+ 1) Get Networks Data from an external http(s) resource
+ 2) Find current config id and generation through GetConfigItem
+ 3) Send ModifyEvent using step 2 information.
+ 4) Schedule WakeUp event on success
+
+ In case of any error algorithm falls back to Initing stage and schedules WakeUp.
+*/
+
+class NetClassifierUpdater : public TActorBootstrapped<NetClassifierUpdater> {
+private:
+ using TBase = TActorBootstrapped<NetClassifierUpdater>;
+
+public:
NetClassifierUpdater(TActorId localConsole)
- : LocalConsole(localConsole)
- {
- }
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::NET_CLASSIFIER_UPDATER;
- }
-
+ : LocalConsole(localConsole)
+ {
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::NET_CLASSIFIER_UPDATER;
+ }
+
void Bootstrap() {
- Become(&TThis::Initing);
+ Become(&TThis::Initing);
Send(SelfId(), new TEvents::TEvWakeup);
- }
-
-private:
- const auto& UpdaterConfig() const {
+ }
+
+private:
+ const auto& UpdaterConfig() const {
return AppData()->NetClassifierConfig.GetUpdaterConfig();
- }
-
- void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) {
- // start the config check
- RequestCurrentConfigViaCookie();
- }
-
- void RequestCurrentConfigViaCookie() {
+ }
+
+ void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) {
+ // start the config check
+ RequestCurrentConfigViaCookie();
+ }
+
+ void RequestCurrentConfigViaCookie() {
BLOG_D("NetClassifierUpdater requested distributable config item via cookie");
-
- auto event = MakeHolder<TEvConsole::TEvGetConfigItemsRequest>();
-
- event->Record.AddItemKinds(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem));
- event->Record.MutableCookieFilter()->AddCookies(COOKIE);
-
- Send(LocalConsole, event.Release());
- }
-
- void InitDefaultConfiguration() {
+
+ auto event = MakeHolder<TEvConsole::TEvGetConfigItemsRequest>();
+
+ event->Record.AddItemKinds(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem));
+ event->Record.MutableCookieFilter()->AddCookies(COOKIE);
+
+ Send(LocalConsole, event.Release());
+ }
+
+ void InitDefaultConfiguration() {
LOG_INFO_S(*TlsActivationContext, NKikimrServices::CMS_CONFIGS,
- "NetClassifierUpdate is adding distributable config item with cookie");
-
- auto event = MakeHolder<TEvConsole::TEvConfigureRequest>();
-
- auto& configItem = *event->Record.AddActions()->MutableAddConfigItem()->MutableConfigItem();
- configItem.MutableConfig()->MutableNetClassifierDistributableConfig(); // just initialize the field
-
- configItem.SetKind(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem));
-
- configItem.SetOrder(CONFIG_ORDER); // prevents config item duplicates
- configItem.SetCookie(COOKIE);
-
- Send(LocalConsole, event.Release());
- }
-
- void HandleWhileIniting(TEvConsole::TEvConfigureResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
+ "NetClassifierUpdate is adding distributable config item with cookie");
+
+ auto event = MakeHolder<TEvConsole::TEvConfigureRequest>();
+
+ auto& configItem = *event->Record.AddActions()->MutableAddConfigItem()->MutableConfigItem();
+ configItem.MutableConfig()->MutableNetClassifierDistributableConfig(); // just initialize the field
+
+ configItem.SetKind(static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem));
+
+ configItem.SetOrder(CONFIG_ORDER); // prevents config item duplicates
+ configItem.SetCookie(COOKIE);
+
+ Send(LocalConsole, event.Release());
+ }
+
+ void HandleWhileIniting(TEvConsole::TEvConfigureResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
BLOG_D("NetClassifierUpdater created a new distributable config item");
- CompleteInitialization();
- } else {
+ CompleteInitialization();
+ } else {
BLOG_ERROR("NetClassifierUpdater failed to add config item: " << record.ShortDebugString());
- InitializeAgain();
- }
- }
-
- void HandleWhileIniting(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
- if (record.ConfigItemsSize() == 0) {
- // cookied config item is missing, add it
- InitDefaultConfiguration();
- } else {
- Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie
-
+ InitializeAgain();
+ }
+ }
+
+ void HandleWhileIniting(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
+ if (record.ConfigItemsSize() == 0) {
+ // cookied config item is missing, add it
+ InitDefaultConfiguration();
+ } else {
+ Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie
+
BLOG_D("NetClassifierUpdater found the distributable config via cookie");
-
- CompleteInitialization();
- }
- } else {
+
+ CompleteInitialization();
+ }
+ } else {
BLOG_ERROR("NetClassifierUpdater failed get current distributable config version: " << record.ShortDebugString());
- InitializeAgain();
- }
- }
-
- STATEFN(Initing) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvConsole::TEvConfigureResponse, HandleWhileIniting);
- hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileIniting);
- hFunc(TEvents::TEvWakeup, HandleWhileIniting);
- hFunc(TEvents::TEvPoisonPill, HandlePoison);
- }
- }
-
- void CompleteInitialization() {
+ InitializeAgain();
+ }
+ }
+
+ STATEFN(Initing) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvConsole::TEvConfigureResponse, HandleWhileIniting);
+ hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileIniting);
+ hFunc(TEvents::TEvWakeup, HandleWhileIniting);
+ hFunc(TEvents::TEvPoisonPill, HandlePoison);
+ }
+ }
+
+ void CompleteInitialization() {
BLOG_D("NetClassifierUpdater has been initialized");
-
- Become(&TThis::Working);
+
+ Become(&TThis::Working);
Send(SelfId(), new TEvents::TEvWakeup);
- }
-
- STATEFN(Working) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, HandleWhileWorking);
- hFunc(TEvConsole::TEvConfigureResponse, HandleWhileWorking);
- hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileWorking);
- hFunc(TEvents::TEvWakeup, HandleWhileWorking);
- hFunc(TEvents::TEvPoisonPill, HandlePoison);
- }
- }
-
+ }
+
+ STATEFN(Working) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, HandleWhileWorking);
+ hFunc(TEvConsole::TEvConfigureResponse, HandleWhileWorking);
+ hFunc(TEvConsole::TEvGetConfigItemsResponse, HandleWhileWorking);
+ hFunc(TEvents::TEvWakeup, HandleWhileWorking);
+ hFunc(TEvents::TEvPoisonPill, HandlePoison);
+ }
+ }
+
auto FormNetDataFromJson(TStringBuf jsonData) const {
NKikimrNetClassifier::TNetData netData;
TVector<TString> tagsToFilter(UpdaterConfig().GetNetBoxTags().begin(), UpdaterConfig().GetNetBoxTags().end());
@@ -212,35 +212,35 @@ private:
return netData;
}
- auto FormNetData(TStringBuf tsvData) const {
- NKikimrNetClassifier::TNetData netData;
-
- while (auto record = tsvData.NextTok('\n')) {
- auto& subnet = *netData.AddSubnets();
- subnet.SetMask(TString(record.NextTok('\t')));
- subnet.SetLabel(TString(record.NextTok('\t')));
- }
-
- return netData;
- }
-
- void ScheduleNextUpdate() {
- const TDuration interval = TDuration::Seconds(UpdaterConfig().GetNetDataUpdateIntervalSeconds());
- Schedule(interval, new TEvents::TEvWakeup);
- }
-
- void InitializeAgain() {
- PackedNetData = {};
- LastUpdateTimestamp = {};
- LastUpdateDatetimeUTC = {};
-
- Become(&TThis::Initing);
-
- const TDuration interval = TDuration::Seconds(UpdaterConfig().GetRetryIntervalSeconds());
- Schedule(interval, new TEvents::TEvWakeup);
- }
-
- void HandleWhileWorking(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) {
+ auto FormNetData(TStringBuf tsvData) const {
+ NKikimrNetClassifier::TNetData netData;
+
+ while (auto record = tsvData.NextTok('\n')) {
+ auto& subnet = *netData.AddSubnets();
+ subnet.SetMask(TString(record.NextTok('\t')));
+ subnet.SetLabel(TString(record.NextTok('\t')));
+ }
+
+ return netData;
+ }
+
+ void ScheduleNextUpdate() {
+ const TDuration interval = TDuration::Seconds(UpdaterConfig().GetNetDataUpdateIntervalSeconds());
+ Schedule(interval, new TEvents::TEvWakeup);
+ }
+
+ void InitializeAgain() {
+ PackedNetData = {};
+ LastUpdateTimestamp = {};
+ LastUpdateDatetimeUTC = {};
+
+ Become(&TThis::Initing);
+
+ const TDuration interval = TDuration::Seconds(UpdaterConfig().GetRetryIntervalSeconds());
+ Schedule(interval, new TEvents::TEvWakeup);
+ }
+
+ void HandleWhileWorking(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) {
if (ev->Get()->Error.empty()) {
if (ev->Get()->Response->Status == "200") {
const auto netData = UpdaterConfig().GetFormat() == NKikimrNetClassifier::TNetClassifierUpdaterConfig::TSV
@@ -248,10 +248,10 @@ private:
: FormNetDataFromJson(ev->Get()->Response->Body);
if (netData.SubnetsSize() != 0) {
PackedNetData = PackAcquiredSubnets(netData);
-
+
LastUpdateTimestamp = TActivationContext::Now();
LastUpdateDatetimeUTC = LastUpdateTimestamp.ToRfc822String(); // for viewer
-
+
// To modify the config it's essential to find the current id and generation
RequestCurrentConfigViaCookie();
return;
@@ -263,89 +263,89 @@ private:
}
} else {
BLOG_ERROR("NetClassifierUpdater failed to get subnets: " << ev->Get()->Error);
- }
+ }
InitializeAgain();
- }
-
- void HandleWhileWorking(TEvConsole::TEvConfigureResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
- // hurray! the update is finished
- ScheduleNextUpdate();
- } else {
+ }
+
+ void HandleWhileWorking(TEvConsole::TEvConfigureResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
+ // hurray! the update is finished
+ ScheduleNextUpdate();
+ } else {
BLOG_ERROR("NetClassifierUpdater failed to update distributable config: " << record.ShortDebugString());
- InitializeAgain();
- }
- }
-
- void HandleWhileWorking(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
- Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie
-
- auto event = MakeHolder<TEvConsole::TEvConfigureRequest>();
-
- auto& configItem = *event->Record.AddActions()->MutableModifyConfigItem()->MutableConfigItem();
-
- // copy id, generation and cookie
- configItem.CopyFrom(ev->Get()->Record.GetConfigItems(0));
-
- auto& distributableConfig = *configItem.MutableConfig()->MutableNetClassifierDistributableConfig();
- distributableConfig.SetPackedNetData(PackedNetData);
- distributableConfig.SetLastUpdateDatetimeUTC(LastUpdateDatetimeUTC);
- distributableConfig.SetLastUpdateTimestamp(LastUpdateTimestamp.MicroSeconds());
-
- Send(LocalConsole, event.Release());
- } else {
+ InitializeAgain();
+ }
+ }
+
+ void HandleWhileWorking(TEvConsole::TEvGetConfigItemsResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ if (record.GetStatus().GetCode() == Ydb::StatusIds::SUCCESS) {
+ Y_VERIFY(record.ConfigItemsSize() == 1); // only one config item should have the cookie
+
+ auto event = MakeHolder<TEvConsole::TEvConfigureRequest>();
+
+ auto& configItem = *event->Record.AddActions()->MutableModifyConfigItem()->MutableConfigItem();
+
+ // copy id, generation and cookie
+ configItem.CopyFrom(ev->Get()->Record.GetConfigItems(0));
+
+ auto& distributableConfig = *configItem.MutableConfig()->MutableNetClassifierDistributableConfig();
+ distributableConfig.SetPackedNetData(PackedNetData);
+ distributableConfig.SetLastUpdateDatetimeUTC(LastUpdateDatetimeUTC);
+ distributableConfig.SetLastUpdateTimestamp(LastUpdateTimestamp.MicroSeconds());
+
+ Send(LocalConsole, event.Release());
+ } else {
BLOG_ERROR("NetClassifierUpdater failed to get current distributable config version: " << record.ShortDebugString());
- InitializeAgain();
- }
- }
-
- TString PackAcquiredSubnets(const NKikimrNetClassifier::TNetData& netData) const {
- TString serializedProto;
+ InitializeAgain();
+ }
+ }
+
+ TString PackAcquiredSubnets(const NKikimrNetClassifier::TNetData& netData) const {
+ TString serializedProto;
Y_PROTOBUF_SUPPRESS_NODISCARD netData.SerializeToString(&serializedProto);
-
- return PackNetData(serializedProto);
- }
-
- void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) {
- // it's high time to update the networks data!
- if (!HttpProxyId) {
- HttpProxyId = Register(NHttp::CreateHttpProxy(HttpSensors));
- }
-
- NHttp::THttpOutgoingRequestPtr httpRequest =
- NHttp::THttpOutgoingRequest::CreateRequestGet(UpdaterConfig().GetNetDataSourceUrl());
-
- Send(HttpProxyId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest));
- }
-
- void HandlePoison(TEvents::TEvPoisonPill::TPtr&) {
- PassAway();
- }
-
-private:
+
+ return PackNetData(serializedProto);
+ }
+
+ void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) {
+ // it's high time to update the networks data!
+ if (!HttpProxyId) {
+ HttpProxyId = Register(NHttp::CreateHttpProxy(HttpSensors));
+ }
+
+ NHttp::THttpOutgoingRequestPtr httpRequest =
+ NHttp::THttpOutgoingRequest::CreateRequestGet(UpdaterConfig().GetNetDataSourceUrl());
+
+ Send(HttpProxyId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest));
+ }
+
+ void HandlePoison(TEvents::TEvPoisonPill::TPtr&) {
+ PassAway();
+ }
+
+private:
TActorId LocalConsole;
TActorId HttpProxyId;
NMonitoring::TMetricRegistry HttpSensors;
-
- TString PackedNetData;
- TString LastUpdateDatetimeUTC;
- TInstant LastUpdateTimestamp;
-};
-
+
+ TString PackedNetData;
+ TString LastUpdateDatetimeUTC;
+ TInstant LastUpdateTimestamp;
+};
+
IActor* MakeNetClassifierUpdaterActor(TActorId localConsole) {
- return new NetClassifierUpdater(localConsole);
-}
-
-TString UnpackNetData(const TString& packedNetData) {
- TStringInput stream(packedNetData);
- try {
- return TZLibDecompress(&stream, ZLib::GZip).ReadAll();
- } catch (const yexception& e) {
- return {};
- }
-}
-
-} // namespace NKikimr::NNetClassifierUpdater
+ return new NetClassifierUpdater(localConsole);
+}
+
+TString UnpackNetData(const TString& packedNetData) {
+ TStringInput stream(packedNetData);
+ try {
+ return TZLibDecompress(&stream, ZLib::GZip).ReadAll();
+ } catch (const yexception& e) {
+ return {};
+ }
+}
+
+} // namespace NKikimr::NNetClassifierUpdater
diff --git a/ydb/core/cms/console/net_classifier_updater.h b/ydb/core/cms/console/net_classifier_updater.h
index 135b33977b..b0ebcd63d9 100644
--- a/ydb/core/cms/console/net_classifier_updater.h
+++ b/ydb/core/cms/console/net_classifier_updater.h
@@ -1,12 +1,12 @@
-#pragma once
-
-#include "defs.h"
-#include "console.h"
-
-namespace NKikimr::NNetClassifierUpdater {
-
+#pragma once
+
+#include "defs.h"
+#include "console.h"
+
+namespace NKikimr::NNetClassifierUpdater {
+
IActor* MakeNetClassifierUpdaterActor(TActorId localConsole);
-
-TString UnpackNetData(const TString& packedNetData);
-
-} // namespace NKikimr::NNetClassifierUpdater
+
+TString UnpackNetData(const TString& packedNetData);
+
+} // namespace NKikimr::NNetClassifierUpdater
diff --git a/ydb/core/cms/console/net_classifier_updater_ut.cpp b/ydb/core/cms/console/net_classifier_updater_ut.cpp
index 0561ab742a..da41ee51f2 100644
--- a/ydb/core/cms/console/net_classifier_updater_ut.cpp
+++ b/ydb/core/cms/console/net_classifier_updater_ut.cpp
@@ -3,89 +3,89 @@
#include <ydb/core/cms/console/console.h>
#include <ydb/core/cms/console/net_classifier_updater.h>
#include <ydb/core/testlib/test_client.h>
-
+
#include <library/cpp/actors/http/http_proxy.cpp>
-
+
#include <library/cpp/protobuf/util/is_equal.h>
-
+
#include <library/cpp/testing/unittest/tests_data.h>
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/json/json_writer.h>
-
-#include <util/string/builder.h>
-
-namespace NKikimr::NNetClassifierUpdaterTests {
-
-using namespace NConsole;
-using namespace Tests;
-
+
+#include <util/string/builder.h>
+
+namespace NKikimr::NNetClassifierUpdaterTests {
+
+using namespace NConsole;
+using namespace Tests;
+
using TNetClassifierUpdaterConfig = NKikimrNetClassifier::TNetClassifierUpdaterConfig;
const TString NETWORKS_URI = "/fancy_path/networks.tsv";
-static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) {
- const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: "
- << netData.size() << "\r\n\r\n" << netData;
- NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content);
-
- return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse);
-}
-
-template<typename TDistributableConfProto>
-bool CheckDistributableConfig(const TDistributableConfProto& config, const NKikimrNetClassifier::TNetData& expectedNetData) {
- if (config.GetPackedNetData()) {
- UNIT_ASSERT(config.GetLastUpdateDatetimeUTC());
- UNIT_ASSERT(config.GetLastUpdateTimestamp());
-
- UNIT_ASSERT_STRINGS_EQUAL(
- TInstant::MicroSeconds(config.GetLastUpdateTimestamp()).ToRfc822String(),
- config.GetLastUpdateDatetimeUTC()
- );
-
- const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(config.GetPackedNetData());
- UNIT_ASSERT(serializedNetData);
-
- NKikimrNetClassifier::TNetData netData;
- UNIT_ASSERT(netData.ParseFromString(serializedNetData));
-
- UNIT_ASSERT(NProtoBuf::IsEqual(netData, expectedNetData));
-
- return true;
- }
-
- return false;
-}
-
-static NKikimrNetClassifier::TNetData FormNetData() {
- NKikimrNetClassifier::TNetData netData;
-
- {
- auto& subnet = *netData.AddSubnets();
- subnet.SetMask("2a02:6b8:fc0a::/48");
- subnet.SetLabel("SAS");
- }
- {
- auto& subnet = *netData.AddSubnets();
- subnet.SetMask("87.250.239.224/31");
- subnet.SetLabel("VLA");
- }
-
- return netData;
-}
-
-static TString ConvertToTsv(const NKikimrNetClassifier::TNetData& netData) {
- TStringBuilder builder;
-
- for (size_t i = 0; i < netData.SubnetsSize(); ++i) {
- const auto& subnet = netData.GetSubnets(i);
- if (i) {
- builder << "\n";
- }
- builder << subnet.GetMask() << "\t" << subnet.GetLabel();
- }
-
- return builder;
-}
-
+static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) {
+ const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: "
+ << netData.size() << "\r\n\r\n" << netData;
+ NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content);
+
+ return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse);
+}
+
+template<typename TDistributableConfProto>
+bool CheckDistributableConfig(const TDistributableConfProto& config, const NKikimrNetClassifier::TNetData& expectedNetData) {
+ if (config.GetPackedNetData()) {
+ UNIT_ASSERT(config.GetLastUpdateDatetimeUTC());
+ UNIT_ASSERT(config.GetLastUpdateTimestamp());
+
+ UNIT_ASSERT_STRINGS_EQUAL(
+ TInstant::MicroSeconds(config.GetLastUpdateTimestamp()).ToRfc822String(),
+ config.GetLastUpdateDatetimeUTC()
+ );
+
+ const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(config.GetPackedNetData());
+ UNIT_ASSERT(serializedNetData);
+
+ NKikimrNetClassifier::TNetData netData;
+ UNIT_ASSERT(netData.ParseFromString(serializedNetData));
+
+ UNIT_ASSERT(NProtoBuf::IsEqual(netData, expectedNetData));
+
+ return true;
+ }
+
+ return false;
+}
+
+static NKikimrNetClassifier::TNetData FormNetData() {
+ NKikimrNetClassifier::TNetData netData;
+
+ {
+ auto& subnet = *netData.AddSubnets();
+ subnet.SetMask("2a02:6b8:fc0a::/48");
+ subnet.SetLabel("SAS");
+ }
+ {
+ auto& subnet = *netData.AddSubnets();
+ subnet.SetMask("87.250.239.224/31");
+ subnet.SetLabel("VLA");
+ }
+
+ return netData;
+}
+
+static TString ConvertToTsv(const NKikimrNetClassifier::TNetData& netData) {
+ TStringBuilder builder;
+
+ for (size_t i = 0; i < netData.SubnetsSize(); ++i) {
+ const auto& subnet = netData.GetSubnets(i);
+ if (i) {
+ builder << "\n";
+ }
+ builder << subnet.GetMask() << "\t" << subnet.GetLabel();
+ }
+
+ return builder;
+}
+
static TString ConvertToJson(const NKikimrNetClassifier::TNetData& netData) {
@@ -126,7 +126,7 @@ NKikimrNetClassifier::TNetClassifierUpdaterConfig CreateUpdaterConfig(
return updaterConfig;
}
-Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
+Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
void TestGetUpdatesFromHttpServer(
const TString& sourceResponce,
const NKikimrNetClassifier::TNetData& expectedNetData,
@@ -134,54 +134,54 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
const TVector<TString>& netBoxTags = {}
) {
NMonitoring::TMetricRegistry sensors;
-
- TPortManager pm;
- const ui16 port = pm.GetPort(2134);
- const ui64 netDataSourcePort = pm.GetPort(13334);
- TServerSettings settings(port);
- auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig();
+
+ TPortManager pm;
+ const ui16 port = pm.GetPort(2134);
+ const ui64 netDataSourcePort = pm.GetPort(13334);
+ TServerSettings settings(port);
+ auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig();
updaterConfig = CreateUpdaterConfig(netDataSourcePort, format, netBoxTags);
- TServer cleverServer = TServer(settings);
- auto& actorSystem = *cleverServer.GetRuntime();
-
- NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors);
+ TServer cleverServer = TServer(settings);
+ auto& actorSystem = *cleverServer.GetRuntime();
+
+ NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors);
NActors::TActorId proxyId = actorSystem.Register(proxy);
-
- actorSystem.Send(
+
+ actorSystem.Send(
new NActors::IEventHandle(proxyId, TActorId(), new NHttp::TEvHttpProxy::TEvAddListeningPort(netDataSourcePort)), 0, true
- );
-
+ );
+
NActors::TActorId serverId = actorSystem.AllocateEdgeActor();
-
+
actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(NETWORKS_URI, serverId)), 0, true);
- TAutoPtr<NActors::IEventHandle> handle;
- NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle);
+ TAutoPtr<NActors::IEventHandle> handle;
+ NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle);
UNIT_ASSERT_EQUAL(request->Request->URL, NETWORKS_URI);
-
+
actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, sourceResponce)), 0, true);
const TActorId sender = actorSystem.AllocateEdgeActor();
-
- size_t iterations = 0;
- while (true) {
- UNIT_ASSERT(++iterations < 60);
-
- const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem);
- actorSystem.Send(
- new IEventHandle(MakeConfigsDispatcherID(sender.NodeId()), sender,
- new TEvConfigsDispatcher::TEvGetConfigRequest(kind)
- ));
-
- const auto event = cleverServer.GetRuntime()->GrabEdgeEvent<TEvConfigsDispatcher::TEvGetConfigResponse>(handle);
-
+
+ size_t iterations = 0;
+ while (true) {
+ UNIT_ASSERT(++iterations < 60);
+
+ const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem);
+ actorSystem.Send(
+ new IEventHandle(MakeConfigsDispatcherID(sender.NodeId()), sender,
+ new TEvConfigsDispatcher::TEvGetConfigRequest(kind)
+ ));
+
+ const auto event = cleverServer.GetRuntime()->GrabEdgeEvent<TEvConfigsDispatcher::TEvGetConfigResponse>(handle);
+
if (CheckDistributableConfig(event->Config->GetNetClassifierDistributableConfig(), expectedNetData)) {
- break;
- }
-
- // wait for the proper update
- Sleep(TDuration::Seconds(1));
- }
- }
+ break;
+ }
+
+ // wait for the proper update
+ Sleep(TDuration::Seconds(1));
+ }
+ }
Y_UNIT_TEST(TestGetUpdatesFromHttpServer) {
auto netData = FormNetData();
@@ -252,6 +252,6 @@ Y_UNIT_TEST_SUITE(TNetClassifierUpdaterTest) {
TestGetUpdatesFromHttpServer(netboxResponce, data, TNetClassifierUpdaterConfig::NETBOX, {"qwerty"});
}
}
-}
-
-} // namespace NKikimr::NNetClassifierUpdaterTests
+}
+
+} // namespace NKikimr::NNetClassifierUpdaterTests
diff --git a/ydb/core/cms/console/ut/ya.make b/ydb/core/cms/console/ut/ya.make
index 377edf66d4..9f42e18427 100644
--- a/ydb/core/cms/console/ut/ya.make
+++ b/ydb/core/cms/console/ut/ya.make
@@ -34,7 +34,7 @@ SRCS(
immediate_controls_configurator_ut.cpp
log_settings_configurator_ut.cpp
modifications_validator_ut.cpp
- net_classifier_updater_ut.cpp
+ net_classifier_updater_ut.cpp
)
END()
diff --git a/ydb/core/cms/console/ya.make b/ydb/core/cms/console/ya.make
index 8f8144a027..e33d3fc19a 100644
--- a/ydb/core/cms/console/ya.make
+++ b/ydb/core/cms/console/ya.make
@@ -59,7 +59,7 @@ SRCS(
log_settings_configurator.h
modifications_validator.cpp
modifications_validator.h
- net_classifier_updater.cpp
+ net_classifier_updater.cpp
shared_cache_configurator.cpp
shared_cache_configurator.h
tx_processor.cpp
diff --git a/ydb/core/driver_lib/cli_utils/cli.h b/ydb/core/driver_lib/cli_utils/cli.h
index a85a38220a..099664792e 100644
--- a/ydb/core/driver_lib/cli_utils/cli.h
+++ b/ydb/core/driver_lib/cli_utils/cli.h
@@ -34,7 +34,7 @@ namespace NDriverClient {
int KeyValueRequest(TCommandConfig &cmdConf, int argc, char **argv);
int PersQueueRequest(TCommandConfig &cmdConf, int argc, char **argv);
int PersQueueStress(TCommandConfig &cmdConf, int argc, char **argv);
- int PersQueueDiscoverClustersRequest(TCommandConfig &cmdConf, int argc, char **argv);
+ int PersQueueDiscoverClustersRequest(TCommandConfig &cmdConf, int argc, char **argv);
int LoadRequest(TCommandConfig &cmdConf, int argc, char **argv);
int ActorsysPerfTest(TCommandConfig &cmdConf, int argc, char **argv);
void HideOptions(NLastGetopt::TOpts& opts, const TString& prefix);
diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp
index 0ba9640b2d..0497d4ea8f 100644
--- a/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp
+++ b/ydb/core/driver_lib/cli_utils/cli_cmds_console.cpp
@@ -119,8 +119,8 @@ public:
ConfigFiles["FeatureFlags"] = "feature_flags.txt";
ConfigFiles["SqsConfig"] = "sqs.txt";
ConfigFiles["PQConfig"] = "pq.txt";
- ConfigFiles["PQClusterDiscoveryConfig"] = "pqcd.txt";
- ConfigFiles["NetClassifierConfig"] = "netclassifier.txt";
+ ConfigFiles["PQClusterDiscoveryConfig"] = "pqcd.txt";
+ ConfigFiles["NetClassifierConfig"] = "netclassifier.txt";
ConfigFiles["KeyConfig"] = "key.txt";
ConfigFiles["PDiskKeyConfig"] = "pdisk_key.txt";
ConfigFiles["ClusterYamlConfig"] = "cluster.yaml";
diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
index 9c5aa29c66..6766dd3171 100644
--- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
+++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
@@ -209,8 +209,8 @@ protected:
config.Opts->AddLongOption("incrhuge-file", "incremental huge blob keeper config file").OptionalArgument("PATH");
config.Opts->AddLongOption("memorylog-file", "set buffer size for memory log").OptionalArgument("PATH");
config.Opts->AddLongOption("pq-file", "PersQueue config file").OptionalArgument("PATH");
- config.Opts->AddLongOption("pqcd-file", "PersQueue cluster discovery config file").OptionalArgument("PATH");
- config.Opts->AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH");
+ config.Opts->AddLongOption("pqcd-file", "PersQueue cluster discovery config file").OptionalArgument("PATH");
+ config.Opts->AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH");
config.Opts->AddLongOption("auth-file", "authorization configuration").OptionalArgument("PATH");
config.Opts->AddLongOption("auth-token-file", "authorization token configuration").OptionalArgument("PATH");
config.Opts->AddLongOption("key-file", "tanant encryption key configuration").OptionalArgument("PATH");
@@ -422,8 +422,8 @@ protected:
OPTION("dyn-nodes-file", DynamicNameserviceConfig);
OPTION("cms-file", CmsConfig);
OPTION("pq-file", PQConfig);
- OPTION("pqcd-file", PQClusterDiscoveryConfig);
- OPTION("netclassifier-file", NetClassifierConfig);
+ OPTION("pqcd-file", PQClusterDiscoveryConfig);
+ OPTION("netclassifier-file", NetClassifierConfig);
OPTION("auth-file", AuthConfig);
OPTION_MERGE("auth-token-file", AuthConfig);
OPTION("key-file", KeyConfig);
diff --git a/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp b/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp
index 01f3f12345..79800a67da 100644
--- a/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp
+++ b/ydb/core/driver_lib/cli_utils/cli_persqueue_cluster_discovery.cpp
@@ -1,87 +1,87 @@
-#include "cli.h"
-
+#include "cli.h"
+
#include <ydb/core/driver_lib/cli_base/cli_grpc.h>
-
+
#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h>
-
+
#include <ydb/public/api/grpc/ydb_operation_v1.grpc.pb.h>
#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h>
-
+
#include <library/cpp/protobuf/json/proto2json.h>
-
-
-namespace NKikimr::NDriverClient {
-
-struct TCmdPersQueueDiscoverWriteSessionClustersConfig final : public TCliCmdConfig {
- void Parse(int argc, char** argv) {
- using namespace NLastGetopt;
-
- TOpts opts = TOpts::Default();
-
- opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic);
- opts.AddLongOption("source-id", "source id").Required().StoreResult(&SourceId);
- opts.AddLongOption("partition-group", "partition group").Optional().StoreResult(&PartitionGroup);
- opts.AddLongOption("preferred-cluster", "prioritized cluster's name").Optional().StoreResult(&PreferredCluster);
-
- opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose);
-
- ConfigureBaseLastGetopt(opts);
-
- TOptsParseResult res(&opts, argc, argv);
- ConfigureMsgBusLastGetopt(res, argc, argv);
- }
-
- TString Topic;
- TString SourceId;
- ui32 PartitionGroup = 0;
- TString PreferredCluster;
-
- bool IsVerbose = false;
-};
-
-struct TCmdPersQueueDiscoverReadSessionClustersConfig final : public TCliCmdConfig {
- void Parse(int argc, char** argv) {
- using namespace NLastGetopt;
-
- TOpts opts = TOpts::Default();
-
- opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic);
- opts.AddLongOption("mirror-to-cluster", "do not set the option to enable all-original mode").Optional().StoreResult(&MirrorToCluster);
-
- opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose);
-
- ConfigureBaseLastGetopt(opts);
-
- TOptsParseResult res(&opts, argc, argv);
- ConfigureMsgBusLastGetopt(res, argc, argv);
- }
-
- TString Topic;
- TString MirrorToCluster;
-
- bool IsVerbose = false;
-};
-
+
+
+namespace NKikimr::NDriverClient {
+
+struct TCmdPersQueueDiscoverWriteSessionClustersConfig final : public TCliCmdConfig {
+ void Parse(int argc, char** argv) {
+ using namespace NLastGetopt;
+
+ TOpts opts = TOpts::Default();
+
+ opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic);
+ opts.AddLongOption("source-id", "source id").Required().StoreResult(&SourceId);
+ opts.AddLongOption("partition-group", "partition group").Optional().StoreResult(&PartitionGroup);
+ opts.AddLongOption("preferred-cluster", "prioritized cluster's name").Optional().StoreResult(&PreferredCluster);
+
+ opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose);
+
+ ConfigureBaseLastGetopt(opts);
+
+ TOptsParseResult res(&opts, argc, argv);
+ ConfigureMsgBusLastGetopt(res, argc, argv);
+ }
+
+ TString Topic;
+ TString SourceId;
+ ui32 PartitionGroup = 0;
+ TString PreferredCluster;
+
+ bool IsVerbose = false;
+};
+
+struct TCmdPersQueueDiscoverReadSessionClustersConfig final : public TCliCmdConfig {
+ void Parse(int argc, char** argv) {
+ using namespace NLastGetopt;
+
+ TOpts opts = TOpts::Default();
+
+ opts.AddLongOption("topic", "topic").Required().StoreResult(&Topic);
+ opts.AddLongOption("mirror-to-cluster", "do not set the option to enable all-original mode").Optional().StoreResult(&MirrorToCluster);
+
+ opts.AddLongOption("verbose", "output detailed information for debugging").Optional().NoArgument().SetFlag(&IsVerbose);
+
+ ConfigureBaseLastGetopt(opts);
+
+ TOptsParseResult res(&opts, argc, argv);
+ ConfigureMsgBusLastGetopt(res, argc, argv);
+ }
+
+ TString Topic;
+ TString MirrorToCluster;
+
+ bool IsVerbose = false;
+};
+
static void PrintGRpcConfigAndRequest(const NGrpc::TGRpcClientConfig& grpcConfig, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest& request) {
- Cerr << "Request endpoint: " << grpcConfig.Locator << " "
- << "timeout: " << grpcConfig.Timeout << " "
- << "max message size: " << grpcConfig.MaxMessageSize << Endl;
-
- Cerr << "Request message: " << NProtobufJson::Proto2Json(request, {}) << Endl;
-}
-
+ Cerr << "Request endpoint: " << grpcConfig.Locator << " "
+ << "timeout: " << grpcConfig.Timeout << " "
+ << "max message size: " << grpcConfig.MaxMessageSize << Endl;
+
+ Cerr << "Request message: " << NProtobufJson::Proto2Json(request, {}) << Endl;
+}
+
template <class TConfig>
static int MakeRequest(const TConfig& config, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest& request) {
- if (config.ClientConfig.Defined()) {
+ if (config.ClientConfig.Defined()) {
if (std::holds_alternative<NGrpc::TGRpcClientConfig>(*config.ClientConfig)) {
const auto& grpcConfig = std::get<NGrpc::TGRpcClientConfig>(*config.ClientConfig);
-
- if (config.IsVerbose) {
- PrintGRpcConfigAndRequest(grpcConfig, request);
- }
-
- Ydb::Operations::Operation response;
+
+ if (config.IsVerbose) {
+ PrintGRpcConfigAndRequest(grpcConfig, request);
+ }
+
+ Ydb::Operations::Operation response;
const int res = DoGRpcRequest<Ydb::PersQueue::V1::ClusterDiscoveryService,
Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest,
Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse,
@@ -90,67 +90,67 @@ static int MakeRequest(const TConfig& config, const Ydb::PersQueue::ClusterDisco
request,
response,
&Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters, "");
-
- if (res == 0) {
+
+ if (res == 0) {
Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult discoveryResult;
- Y_VERIFY(response.result().UnpackTo(&discoveryResult));
-
- Y_VERIFY(response.ready()); // there's nothing to wait for
-
- Cerr << "Code: " << ToString(response.status()) << ". Result message: " << NProtobufJson::Proto2Json(discoveryResult, {}) << Endl;
- return 0;
- }
- }
-
- }
-
- return 1;
-}
-
-int PersQueueDiscoverClustersRequest(TCommandConfig&, int argc, char** argv) {
- Y_VERIFY(argc > 2);
-
- if (argv[1] == TStringBuf("write-session")) {
- TCmdPersQueueDiscoverWriteSessionClustersConfig config;
- config.Parse(argc - 1, argv + 1);
-
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request;
- auto* writeSessionParams = request.add_write_sessions();
-
- writeSessionParams->set_topic(config.Topic);
- writeSessionParams->set_source_id(config.SourceId);
- writeSessionParams->set_partition_group(config.PartitionGroup);
-
- if (config.PreferredCluster) {
- writeSessionParams->set_preferred_cluster_name(config.PreferredCluster);
- }
-
- request.set_minimal_version(0);
-
+ Y_VERIFY(response.result().UnpackTo(&discoveryResult));
+
+ Y_VERIFY(response.ready()); // there's nothing to wait for
+
+ Cerr << "Code: " << ToString(response.status()) << ". Result message: " << NProtobufJson::Proto2Json(discoveryResult, {}) << Endl;
+ return 0;
+ }
+ }
+
+ }
+
+ return 1;
+}
+
+int PersQueueDiscoverClustersRequest(TCommandConfig&, int argc, char** argv) {
+ Y_VERIFY(argc > 2);
+
+ if (argv[1] == TStringBuf("write-session")) {
+ TCmdPersQueueDiscoverWriteSessionClustersConfig config;
+ config.Parse(argc - 1, argv + 1);
+
+ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request;
+ auto* writeSessionParams = request.add_write_sessions();
+
+ writeSessionParams->set_topic(config.Topic);
+ writeSessionParams->set_source_id(config.SourceId);
+ writeSessionParams->set_partition_group(config.PartitionGroup);
+
+ if (config.PreferredCluster) {
+ writeSessionParams->set_preferred_cluster_name(config.PreferredCluster);
+ }
+
+ request.set_minimal_version(0);
+
return MakeRequest(config, request);
- } else if (argv[1] == TStringBuf("read-session")) {
- TCmdPersQueueDiscoverReadSessionClustersConfig config;
- config.Parse(argc - 1, argv + 1);
-
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request;
- auto* readSessionParams = request.add_read_sessions();
-
- readSessionParams->set_topic(config.Topic);
-
- if (config.MirrorToCluster) {
- readSessionParams->set_mirror_to_cluster(config.MirrorToCluster);
- } else {
- readSessionParams->mutable_all_original(); // set the field
- }
-
- request.set_minimal_version(0);
-
+ } else if (argv[1] == TStringBuf("read-session")) {
+ TCmdPersQueueDiscoverReadSessionClustersConfig config;
+ config.Parse(argc - 1, argv + 1);
+
+ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request;
+ auto* readSessionParams = request.add_read_sessions();
+
+ readSessionParams->set_topic(config.Topic);
+
+ if (config.MirrorToCluster) {
+ readSessionParams->set_mirror_to_cluster(config.MirrorToCluster);
+ } else {
+ readSessionParams->mutable_all_original(); // set the field
+ }
+
+ request.set_minimal_version(0);
+
return MakeRequest(config, request);
- }
-
- Cerr << "Bad commands params" << Endl;
- return 1;
-}
-
-} // namespace NKikimr::NDriverClient
+ }
+
+ Cerr << "Bad commands params" << Endl;
+ return 1;
+}
+
+} // namespace NKikimr::NDriverClient
diff --git a/ydb/core/driver_lib/cli_utils/ya.make b/ydb/core/driver_lib/cli_utils/ya.make
index c90b10db65..cb1f9670be 100644
--- a/ydb/core/driver_lib/cli_utils/ya.make
+++ b/ydb/core/driver_lib/cli_utils/ya.make
@@ -27,7 +27,7 @@ SRCS(
cli_fakeinitshard.cpp
cli_keyvalue.cpp
cli_persqueue.cpp
- cli_persqueue_cluster_discovery.cpp
+ cli_persqueue_cluster_discovery.cpp
cli_persqueue_stress.cpp
cli_load.cpp
cli_minikql_compile_and_exec.cpp
diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h
index 8cafc1f9bf..faf1797413 100644
--- a/ydb/core/driver_lib/run/config.h
+++ b/ydb/core/driver_lib/run/config.h
@@ -53,9 +53,9 @@ union TBasicKikimrServicesMask {
bool EnableSecurityServices:1;
bool EnableTabletInfo:1;
bool EnableQuoterService:1;
- bool EnablePersQueueClusterDiscovery:1;
- bool EnableNetClassifier:1;
- bool EnablePersQueueClusterTracker:1;
+ bool EnablePersQueueClusterDiscovery:1;
+ bool EnableNetClassifier:1;
+ bool EnablePersQueueClusterTracker:1;
bool EnableSysViewService:1;
bool EnableMeteringWriter:1;
bool EnableSchemeBoardMonitoring:1;
diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp
index 160efe3232..bb68656291 100644
--- a/ydb/core/driver_lib/run/config_parser.cpp
+++ b/ydb/core/driver_lib/run/config_parser.cpp
@@ -66,8 +66,8 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts&
opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT");
opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT");
opts.AddLongOption("pq-file", "PQ config file").OptionalArgument("PATH");
- opts.AddLongOption("pqcd-file", "PQCD config file").OptionalArgument("PATH");
- opts.AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH");
+ opts.AddLongOption("pqcd-file", "PQCD config file").OptionalArgument("PATH");
+ opts.AddLongOption("netclassifier-file", "NetClassifier config file").OptionalArgument("PATH");
opts.AddLongOption("auth-file", "authorization config file").OptionalArgument("PATH");
opts.AddLongOption("auth-token-file", "authorization token config file").OptionalArgument("PATH");
opts.AddLongOption("key-file", "encryption key config file").OptionalArgument("PATH");
@@ -172,14 +172,14 @@ void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResu
Y_VERIFY(ParsePBFromFile(res.Get("pq-file"), Config.AppConfig.MutablePQConfig()));
}
- if (res.Has("pqcd-file")) {
- Y_VERIFY(ParsePBFromFile(res.Get("pqcd-file"), Config.AppConfig.MutablePQClusterDiscoveryConfig()));
- }
-
- if (res.Has("netclassifier-file")) {
- Y_VERIFY(ParsePBFromFile(res.Get("netclassifier-file"), Config.AppConfig.MutableNetClassifierConfig()));
- }
-
+ if (res.Has("pqcd-file")) {
+ Y_VERIFY(ParsePBFromFile(res.Get("pqcd-file"), Config.AppConfig.MutablePQClusterDiscoveryConfig()));
+ }
+
+ if (res.Has("netclassifier-file")) {
+ Y_VERIFY(ParsePBFromFile(res.Get("netclassifier-file"), Config.AppConfig.MutableNetClassifierConfig()));
+ }
+
if (res.Has("auth-file")) {
Y_VERIFY(ParsePBFromFile(res.Get("auth-file"), Config.AppConfig.MutableAuthConfig()));
}
diff --git a/ydb/core/driver_lib/run/driver.h b/ydb/core/driver_lib/run/driver.h
index 80cc27cfe9..7ad96718d1 100644
--- a/ydb/core/driver_lib/run/driver.h
+++ b/ydb/core/driver_lib/run/driver.h
@@ -34,7 +34,7 @@ namespace NKikimr {
XX(EDM_KEYVALUE_REQUEST, "keyvalue-request", "send protobuf request to a keyvalue tablet") \
XX(EDM_PERSQUEUE_REQUEST, "persqueue-request", "send protobuf request to a persqueue tablet") \
XX(EDM_PERSQUEUE_STRESS, "persqueue-stress", "stress read or write to a persqueue tablet") \
- XX(EDM_PERSQUEUE_DISCOVER_CLUSTERS, "persqueue-discover-clusters", "persqueue session clusters discovery") \
+ XX(EDM_PERSQUEUE_DISCOVER_CLUSTERS, "persqueue-discover-clusters", "persqueue session clusters discovery") \
XX(EDM_LOAD_REQUEST, "bs-load-test", "send protobuf request to blobstorage test load actor (https://wiki.yandex-team.ru/kikimr/developers/BSLoadTest/)") \
XX(EDM_ACTORSYS_PERFTEST, "actorsys-perf-test", "make actorsystem performance test") \
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 3c4c21858e..819c1478d1 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -51,7 +51,7 @@
#include <ydb/core/kesus/tablet/tablet.h>
#include <ydb/core/keyvalue/keyvalue.h>
-
+
#include <ydb/core/test_tablet/test_tablet.h>
#include <ydb/core/test_tablet/state_server_interface.h>
@@ -142,7 +142,7 @@
#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
#include <library/cpp/actors/protos/services_common.pb.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/actorsystem.h>
#include <library/cpp/actors/core/event_local.h>
@@ -1833,33 +1833,33 @@ void TPersQueueL2CacheInitializer::InitializeServices(NActors::TActorSystemSetup
TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)));
}
-// TNetClassifierInitializer
-
-TNetClassifierInitializer::TNetClassifierInitializer(const TKikimrRunConfig& runConfig)
- : IKikimrServicesInitializer(runConfig)
-{}
-
-void TNetClassifierInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
- IActor* actor = NNetClassifier::CreateNetClassifier();
-
+// TNetClassifierInitializer
+
+TNetClassifierInitializer::TNetClassifierInitializer(const TKikimrRunConfig& runConfig)
+ : IKikimrServicesInitializer(runConfig)
+{}
+
+void TNetClassifierInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
+ IActor* actor = NNetClassifier::CreateNetClassifier();
+
setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(
- NNetClassifier::MakeNetClassifierID(),
- TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)));
-}
-
-// TPersQueueClusterTracker
-
-TPersQueueClusterTrackerInitializer::TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig)
- : IKikimrServicesInitializer(runConfig)
-{}
-
-void TPersQueueClusterTrackerInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
- IActor* actor = NPQ::NClusterTracker::CreateClusterTracker();
+ NNetClassifier::MakeNetClassifierID(),
+ TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)));
+}
+
+// TPersQueueClusterTracker
+
+TPersQueueClusterTrackerInitializer::TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig)
+ : IKikimrServicesInitializer(runConfig)
+{}
+
+void TPersQueueClusterTrackerInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
+ IActor* actor = NPQ::NClusterTracker::CreateClusterTracker();
setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(
- NPQ::NClusterTracker::MakeClusterTrackerID(),
- TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)));
-}
-
+ NPQ::NClusterTracker::MakeClusterTrackerID(),
+ TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)));
+}
+
// TPersQueueLibSharedInstanceInitializer
TPersQueueLibSharedInstanceInitializer::TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig)
@@ -2108,7 +2108,7 @@ void TSqsServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu
NSQS::MakeSqsProxyServiceID(NodeId),
TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId));
}
-
+
Factories->SqsAuthFactory->Initialize(
setup->LocalServices, *appData, Config.GetSqsConfig());
}
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h
index 0bdfc3a238..407ce1bb7b 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.h
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h
@@ -336,20 +336,20 @@ public:
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
};
-class TNetClassifierInitializer : public IKikimrServicesInitializer {
-public:
- TNetClassifierInitializer(const TKikimrRunConfig& runConfig);
+class TNetClassifierInitializer : public IKikimrServicesInitializer {
+public:
+ TNetClassifierInitializer(const TKikimrRunConfig& runConfig);
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
-};
-
-class TPersQueueClusterTrackerInitializer : public IKikimrServicesInitializer {
-public:
- TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig);
+};
+
+class TPersQueueClusterTrackerInitializer : public IKikimrServicesInitializer {
+public:
+ TPersQueueClusterTrackerInitializer(const TKikimrRunConfig& runConfig);
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
-};
-
+};
+
class TPersQueueLibSharedInstanceInitializer : public IKikimrServicesInitializer {
public:
TPersQueueLibSharedInstanceInitializer(const TKikimrRunConfig& runConfig);
diff --git a/ydb/core/driver_lib/run/main.cpp b/ydb/core/driver_lib/run/main.cpp
index 24d9b36cf7..f0a9315685 100644
--- a/ydb/core/driver_lib/run/main.cpp
+++ b/ydb/core/driver_lib/run/main.cpp
@@ -138,8 +138,8 @@ int MainRun(const TKikimrRunConfig& runConfig, std::shared_ptr<TModuleFactories>
return NDriverClient::PersQueueRequest(cmdConf, argc, argv);
case EDM_PERSQUEUE_STRESS:
return NDriverClient::PersQueueStress(cmdConf, argc, argv);
- case EDM_PERSQUEUE_DISCOVER_CLUSTERS:
- return NDriverClient::PersQueueDiscoverClustersRequest(cmdConf, argc, argv);
+ case EDM_PERSQUEUE_DISCOVER_CLUSTERS:
+ return NDriverClient::PersQueueDiscoverClustersRequest(cmdConf, argc, argv);
case EDM_LOAD_REQUEST:
return NDriverClient::LoadRequest(cmdConf, argc, argv);
case EDM_ACTORSYS_PERFTEST:
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index 37d215b0d2..a4f74aa4e0 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -513,7 +513,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) {
names["pq"] = &hasPQ;
bool hasPQv1 = false;
names["pqv1"] = &hasPQv1;
- bool hasPQCD = false;
+ bool hasPQCD = false;
names["pqcd"] = &hasPQCD;
bool hasS3Internal = false;
names["s3_internal"] = &hasS3Internal;
@@ -541,7 +541,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) {
names["auth"] = &hasAuth;
std::unordered_set<TString> enabled;
- for (const auto& name : services) {
+ for (const auto& name : services) {
enabled.insert(name);
}
for (const auto& name : grpcConfig.GetServicesEnabled()) {
@@ -677,8 +677,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) {
server.AddService(new NGRpcService::V1::TGRpcPersQueueService(ActorSystem.Get(), Counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId));
}
- if (hasPQCD) {
- // the service has its own flag since it should be capable of using custom grpc port
+ if (hasPQCD) {
+ // the service has its own flag since it should be capable of using custom grpc port
const auto& pqcdConfig = AppData->PQClusterDiscoveryConfig;
TMaybe<ui64> inflightLimit = Nothing();
if (pqcdConfig.HasRequestInflightLimit() && pqcdConfig.GetRequestInflightLimit() > 0) {
@@ -687,8 +687,8 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) {
server.AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(
ActorSystem.Get(), Counters, grpcRequestProxyId, inflightLimit
));
- }
-
+ }
+
if (hasCms) {
server.AddService(new NGRpcService::TGRpcCmsService(ActorSystem.Get(), Counters, grpcRequestProxyId));
}
@@ -870,33 +870,33 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig)
AppData->LocalScopeId = runConfig.ScopeId;
// setup streaming config
- if (runConfig.AppConfig.GetGRpcConfig().HasStreamingConfig()) {
+ if (runConfig.AppConfig.GetGRpcConfig().HasStreamingConfig()) {
AppData->StreamingConfig.CopyFrom(runConfig.AppConfig.GetGRpcConfig().GetStreamingConfig());
- }
+ }
- if (runConfig.AppConfig.HasPQConfig()) {
+ if (runConfig.AppConfig.HasPQConfig()) {
AppData->PQConfig.CopyFrom(runConfig.AppConfig.GetPQConfig());
- }
-
- if (runConfig.AppConfig.HasPQClusterDiscoveryConfig()) {
- AppData->PQClusterDiscoveryConfig.CopyFrom(runConfig.AppConfig.GetPQClusterDiscoveryConfig());
- }
-
- if (runConfig.AppConfig.HasNetClassifierConfig()) {
- AppData->NetClassifierConfig.CopyFrom(runConfig.AppConfig.GetNetClassifierConfig());
- }
-
- if (runConfig.AppConfig.HasSqsConfig()) {
- AppData->SqsConfig.CopyFrom(runConfig.AppConfig.GetSqsConfig());
- }
-
- if (runConfig.AppConfig.HasAuthConfig()) {
+ }
+
+ if (runConfig.AppConfig.HasPQClusterDiscoveryConfig()) {
+ AppData->PQClusterDiscoveryConfig.CopyFrom(runConfig.AppConfig.GetPQClusterDiscoveryConfig());
+ }
+
+ if (runConfig.AppConfig.HasNetClassifierConfig()) {
+ AppData->NetClassifierConfig.CopyFrom(runConfig.AppConfig.GetNetClassifierConfig());
+ }
+
+ if (runConfig.AppConfig.HasSqsConfig()) {
+ AppData->SqsConfig.CopyFrom(runConfig.AppConfig.GetSqsConfig());
+ }
+
+ if (runConfig.AppConfig.HasAuthConfig()) {
AppData->AuthConfig.CopyFrom(runConfig.AppConfig.GetAuthConfig());
- }
+ }
- if (runConfig.AppConfig.HasKeyConfig()) {
+ if (runConfig.AppConfig.HasKeyConfig()) {
AppData->KeyConfig.CopyFrom(runConfig.AppConfig.GetKeyConfig());
- }
+ }
if (runConfig.AppConfig.HasPDiskKeyConfig()) {
AppData->PDiskKeyConfig.CopyFrom(runConfig.AppConfig.GetPDiskKeyConfig());
@@ -1103,12 +1103,12 @@ void TKikimrRunner::InitializeActorSystem(
}
}
- if (servicesMask.EnableSqs && AppData->SqsConfig.GetEnableSqs()) {
- if (AppData->SqsConfig.GetHttpServerConfig().GetPort()) {
-
- SqsHttp.Reset(new NSQS::TAsyncHttpServer(AppData->SqsConfig));
+ if (servicesMask.EnableSqs && AppData->SqsConfig.GetEnableSqs()) {
+ if (AppData->SqsConfig.GetHttpServerConfig().GetPort()) {
+
+ SqsHttp.Reset(new NSQS::TAsyncHttpServer(AppData->SqsConfig));
SqsHttp->Initialize(ActorSystem.Get(),
- GetServiceCounters(AppData->Counters, "sqs"),
+ GetServiceCounters(AppData->Counters, "sqs"),
GetServiceCounters(AppData->Counters, "ymq_public"),
AppData->UserPoolId);
}
@@ -1253,12 +1253,12 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
if (serviceMask.EnablePersQueueL2Cache) {
sil->AddServiceInitializer(new TPersQueueL2CacheInitializer(runConfig));
}
- if (serviceMask.EnableNetClassifier) {
- sil->AddServiceInitializer(new TNetClassifierInitializer(runConfig));
- }
- if (serviceMask.EnablePersQueueClusterTracker) {
- sil->AddServiceInitializer(new TPersQueueClusterTrackerInitializer(runConfig));
- }
+ if (serviceMask.EnableNetClassifier) {
+ sil->AddServiceInitializer(new TNetClassifierInitializer(runConfig));
+ }
+ if (serviceMask.EnablePersQueueClusterTracker) {
+ sil->AddServiceInitializer(new TPersQueueClusterTrackerInitializer(runConfig));
+ }
sil->AddServiceInitializer(new TPersQueueLibSharedInstanceInitializer(runConfig));
diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h
index 69a88b388f..44b25c4a5f 100644
--- a/ydb/core/grpc_services/base/base.h
+++ b/ydb/core/grpc_services/base/base.h
@@ -104,7 +104,7 @@ struct TRpcServices {
EvPQReadInfo,
EvListOperations,
EvExportToYt,
- EvDiscoverPQClusters,
+ EvDiscoverPQClusters,
EvBulkUpsert,
EvWhoAmI,
EvKikhouseDescribeTable,
diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h
index b4bc022404..97315f6e9f 100644
--- a/ydb/core/grpc_services/grpc_request_proxy.h
+++ b/ydb/core/grpc_services/grpc_request_proxy.h
@@ -100,7 +100,7 @@ protected:
void Handle(TEvExportToS3Request::TPtr& ev, const TActorContext& ctx);
void Handle(TEvImportFromS3Request::TPtr& ev, const TActorContext& ctx);
void Handle(TEvImportDataRequest::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvBulkUpsertRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvWhoAmIRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvCreateRateLimiterResource::TPtr& ev, const TActorContext& ctx);
diff --git a/ydb/core/mind/address_classification/counters.cpp b/ydb/core/mind/address_classification/counters.cpp
index 1aa6b04759..7491c4251e 100644
--- a/ydb/core/mind/address_classification/counters.cpp
+++ b/ydb/core/mind/address_classification/counters.cpp
@@ -1,19 +1,19 @@
-#include "counters.h"
-
-namespace NKikimr::NNetClassifier::NCounters {
-
-#define SETUP_SIMPLE_COUNTER(name, derived) \
- name(counters->GetCounter(#name, derived))
-
-TNetClassifierCounters::TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters)
- : SETUP_SIMPLE_COUNTER(GoodConfigNotificationsCount, true)
- , SETUP_SIMPLE_COUNTER(BrokenConfigNotificationsCount, true)
- , SETUP_SIMPLE_COUNTER(SubscribersCount, false)
- , SETUP_SIMPLE_COUNTER(NetDataSourceType, false)
- , SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false)
-{
-}
-
-#undef SETUP_SIMPLE_COUNTER
-
-} // namespace NKikimr::NNetClassifier::NCounters
+#include "counters.h"
+
+namespace NKikimr::NNetClassifier::NCounters {
+
+#define SETUP_SIMPLE_COUNTER(name, derived) \
+ name(counters->GetCounter(#name, derived))
+
+TNetClassifierCounters::TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters)
+ : SETUP_SIMPLE_COUNTER(GoodConfigNotificationsCount, true)
+ , SETUP_SIMPLE_COUNTER(BrokenConfigNotificationsCount, true)
+ , SETUP_SIMPLE_COUNTER(SubscribersCount, false)
+ , SETUP_SIMPLE_COUNTER(NetDataSourceType, false)
+ , SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false)
+{
+}
+
+#undef SETUP_SIMPLE_COUNTER
+
+} // namespace NKikimr::NNetClassifier::NCounters
diff --git a/ydb/core/mind/address_classification/counters.h b/ydb/core/mind/address_classification/counters.h
index bafab67abc..0ca6e92468 100644
--- a/ydb/core/mind/address_classification/counters.h
+++ b/ydb/core/mind/address_classification/counters.h
@@ -1,26 +1,26 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/monlib/dynamic_counters/counters.h>
-
-#include <util/generic/ptr.h>
-#include <util/generic/noncopyable.h>
-
-namespace NKikimr::NNetClassifier::NCounters {
-
-using namespace NMonitoring;
-
-struct TNetClassifierCounters : TAtomicRefCount<TNetClassifierCounters>, TNonCopyable {
- using TPtr = TIntrusivePtr<TNetClassifierCounters>;
- using TCounterPtr = TDynamicCounters::TCounterPtr;
-
- TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters);
-
- TCounterPtr GoodConfigNotificationsCount;
- TCounterPtr BrokenConfigNotificationsCount;
- TCounterPtr SubscribersCount;
- TCounterPtr NetDataSourceType;
-
- TCounterPtr NetDataUpdateLagSeconds;
-};
-
-} // namespace NKikimr::NNetClassifier::NCounters
+
+#include <util/generic/ptr.h>
+#include <util/generic/noncopyable.h>
+
+namespace NKikimr::NNetClassifier::NCounters {
+
+using namespace NMonitoring;
+
+struct TNetClassifierCounters : TAtomicRefCount<TNetClassifierCounters>, TNonCopyable {
+ using TPtr = TIntrusivePtr<TNetClassifierCounters>;
+ using TCounterPtr = TDynamicCounters::TCounterPtr;
+
+ TNetClassifierCounters(TIntrusivePtr<TDynamicCounters> counters);
+
+ TCounterPtr GoodConfigNotificationsCount;
+ TCounterPtr BrokenConfigNotificationsCount;
+ TCounterPtr SubscribersCount;
+ TCounterPtr NetDataSourceType;
+
+ TCounterPtr NetDataUpdateLagSeconds;
+};
+
+} // namespace NKikimr::NNetClassifier::NCounters
diff --git a/ydb/core/mind/address_classification/net_classifier.cpp b/ydb/core/mind/address_classification/net_classifier.cpp
index d6ef5c7658..11c7e6750c 100644
--- a/ydb/core/mind/address_classification/net_classifier.cpp
+++ b/ydb/core/mind/address_classification/net_classifier.cpp
@@ -1,416 +1,416 @@
-#include "counters.h"
-#include "net_classifier.h"
-
+#include "counters.h"
+#include "net_classifier.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/counters.h>
-
+
#include <ydb/core/cms/console/configs_dispatcher.h>
#include <ydb/core/cms/console/console.h>
#include <ydb/core/cms/console/net_classifier_updater.h>
#include <ydb/core/mon/mon.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
-
-#include <util/generic/hash.h>
-#include <util/stream/file.h>
-#include <util/system/fs.h>
-#include <util/system/fstat.h>
-
-#include <vector>
-
-namespace NKikimr::NNetClassifier {
-
-using namespace NAddressClassifier;
-using namespace NConsole;
-using namespace NCounters;
-
-inline auto& Ctx() {
- return TActivationContext::AsActorContext();
-}
-
-class TNetClassifier: public TActorBootstrapped<TNetClassifier> {
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::NET_CLASSIFIER_ACTOR;
- }
-
- void Bootstrap() {
- Become(&TThis::WaitingForSubscribers);
- }
-
-private:
- static TMaybe<TInstant> GetNetDataFileModTimestamp(const TString& filePath) {
- try {
- TFileStat stat(filePath);
- if (stat.IsFile() && !stat.IsNull()) {
- return TInstant::Seconds(stat.MTime);
- } else {
- ythrow yexception() << "file does not exist: " << filePath;
- }
- } catch (const yexception& ex) {
- LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to get NetData file stats: " << ex.what());
- }
-
- return Nothing();
- }
-
- const auto& Cfg() const {
- return AppData(Ctx())->NetClassifierConfig;
- }
-
+
+#include <util/generic/hash.h>
+#include <util/stream/file.h>
+#include <util/system/fs.h>
+#include <util/system/fstat.h>
+
+#include <vector>
+
+namespace NKikimr::NNetClassifier {
+
+using namespace NAddressClassifier;
+using namespace NConsole;
+using namespace NCounters;
+
+inline auto& Ctx() {
+ return TActivationContext::AsActorContext();
+}
+
+class TNetClassifier: public TActorBootstrapped<TNetClassifier> {
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::NET_CLASSIFIER_ACTOR;
+ }
+
+ void Bootstrap() {
+ Become(&TThis::WaitingForSubscribers);
+ }
+
+private:
+ static TMaybe<TInstant> GetNetDataFileModTimestamp(const TString& filePath) {
+ try {
+ TFileStat stat(filePath);
+ if (stat.IsFile() && !stat.IsNull()) {
+ return TInstant::Seconds(stat.MTime);
+ } else {
+ ythrow yexception() << "file does not exist: " << filePath;
+ }
+ } catch (const yexception& ex) {
+ LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to get NetData file stats: " << ex.what());
+ }
+
+ return Nothing();
+ }
+
+ const auto& Cfg() const {
+ return AppData(Ctx())->NetClassifierConfig;
+ }
+
void AddSubscriber(const TActorId subscriberId) {
- Subscribers.insert(subscriberId);
-
- *Counters->SubscribersCount = Subscribers.size();
- }
-
- STATEFN(WaitingForSubscribers) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWaiting);
- }
- }
-
- void SubscribeForConfigUpdates() {
- const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem);
- Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(kind));
- }
-
- void HandleWhileWaiting(TEvNetClassifier::TEvSubscribe::TPtr& ev) {
- Become(&TThis::Initing);
-
- InitCounters();
-
- AddSubscriber(ev->Sender);
-
- const auto& netDataFilePath = Cfg().GetNetDataFilePath();
- if (netDataFilePath) {
- MaybeNetDataFilePath = netDataFilePath;
-
- MaybeNetDataFileModTs = GetNetDataFileModTimestamp(*MaybeNetDataFilePath);
- }
-
- SubscribeForConfigUpdates();
- }
-
- STATEFN(Initing) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileIniting);
- hFunc(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWhileIniting);
- hFunc(TEvConfigsDispatcher::TEvGetConfigResponse, HandleWhileIniting);
- hFunc(TEvents::TEvWakeup, HandleWhileIniting);
- hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters);
- }
- }
-
- void HandleWhileIniting(TEvNetClassifier::TEvSubscribe::TPtr& ev) {
- AddSubscriber(ev->Sender);
-
- // all subsribers get classifier update during a transition to a Working state
- }
-
- void RequestDistributableConfig() {
- const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem);
- Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvGetConfigRequest(kind));
- }
-
- void HandleWhileIniting(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
- LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "subscribed to ConfigsDispatcher for NetData updates");
-
- // Subscription is set but the console tablet may still be unavailable hence it's a good idea to schedule a timeout event
- Schedule(TDuration::Seconds(Cfg().GetCmsConfigTimeoutSeconds()), new TEvents::TEvWakeup);
-
- RequestDistributableConfig();
-
- // This leads to Case 1 or Case 2
- }
-
- // Case 1: GetConfig request timed out
- void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) {
- LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "TIMEOUT: failed to get distributable config, fall back to file: " << MaybeNetDataFilePath);
- InitFromFile();
- }
-
- template<typename TConfPtr>
- bool ShouldUseDistributableConfigOnInit(const TConfPtr& ptr) const {
- if (!ptr) {
- return false;
- }
-
- if (!ptr->HasNetClassifierDistributableConfig()) {
- return false;
- }
-
- const auto& config = ptr->GetNetClassifierDistributableConfig();
- if (config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp()) {
- if (MaybeNetDataFileModTs) {
- return TInstant::MicroSeconds(config.GetLastUpdateTimestamp()) > *MaybeNetDataFileModTs;
- }
-
- return true;
- }
-
- return false;
- }
-
- // Case 2: GetConfig request succeeded
- void HandleWhileIniting(TEvConfigsDispatcher::TEvGetConfigResponse::TPtr& ev) {
- if (ShouldUseDistributableConfigOnInit(ev->Get()->Config)) {
- LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will initialize from distributable config");
-
- if (UpdateFromDistributableConfig(ev->Get()->Config->GetNetClassifierDistributableConfig())) {
- ReportGoodConfig();
- UpdateNetDataSource(ENetDataSourceType::DistributableConfig);
-
- LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from distributable config");
-
- CompleteInitialization();
- return;
- } else {
- LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from distributed config");
- }
- }
-
- ReportBadConfig();
- LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "distributable config is empty, broken or outdated, will use file: " << MaybeNetDataFilePath);
- InitFromFile();
- }
-
- void InitFromFile() {
- // NetData file may be outdated hence warning log level is used
- LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will try to initialize from file: " << MaybeNetDataFilePath);
- LabeledAddressClassifier = TryReadNetDataFromFile();
- if (LabeledAddressClassifier) {
- NetDataUpdateTimestamp = MaybeNetDataFileModTs;
- UpdateNetDataSource(ENetDataSourceType::File);
-
- LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from file: " << MaybeNetDataFilePath);
- } else {
- LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from file: " << MaybeNetDataFilePath);
- }
-
- CompleteInitialization();
- }
-
- bool CheckDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) const {
- return config.GetPackedNetData() && config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp();
- }
-
- bool UpdateFromDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) {
- if (CheckDistributableConfig(config)) {
- LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got new config with datetime: " << config.GetLastUpdateDatetimeUTC());
-
- auto labeledAddressClassifier = BuildNetClassifierFromPackedNetData(config.GetPackedNetData());
- if (labeledAddressClassifier) {
- LabeledAddressClassifier = std::move(labeledAddressClassifier);
- NetDataUpdateTimestamp = TInstant::MicroSeconds(config.GetLastUpdateTimestamp());
-
- return true;
- } else {
- LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to parse NetData from distributable configuration");
- }
- } else {
- LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got bad distributable configuration");
- }
-
- return false;
- }
-
- void CompleteInitialization() {
- Become(&TThis::Working);
-
- NActors::TMon* mon = AppData(Ctx())->Mon;
- if (mon) {
- NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors");
- mon->RegisterActorPage(page, "netclassifier", "NetClassifier", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID);
- }
-
- BroadcastClassifierUpdate();
- }
-
- STATEFN(Working) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvConsole::TEvConfigNotificationRequest, HandleWhileWorking);
- hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWorking);
- hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
- hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters);
- }
- }
-
- static TString FormatTs(const TMaybe<TInstant>& time) {
- if (time) {
- return time->ToRfc822String();
- }
- return "NULL";
- }
-
- void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev)
- {
- TStringBuilder message;
- message << "NetClassifier info: \n";
- message << "\tUnique subscribers: " << Subscribers.size() << "\n";
- message << "\tClassifier updates sent: " << ClassifierUpdatesSent << "\n";
- message << "\tLast classifier update broadcast ts: " << FormatTs(LastClassifierUpdateBroadcastTs) << "\n";
- message << "\tNetData file path: " << MaybeNetDataFilePath << "\n";
- message << "\tNetData file ts: " << FormatTs(MaybeNetDataFileModTs) << "\n";
- message << "\tCurrent NetData ts: " << FormatTs(NetDataUpdateTimestamp) << "\n";
- message << "\tGood config notifications: " << GoodConfigNotifications << "\n";
- message << "\tBroken or empty config notifications: " << BrokenOrEmptyConfigNotifications << "\n";
- message << "\tCurrent NetData source: " << ToString(CurrentNetDataSource) << "\n";
- message << "\tClassifier: " << (LabeledAddressClassifier ? "OK" : "NULL");
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(NMonitoring::HTTPOKTEXT) + TString(message), 0,
- NMon::IEvHttpInfoRes::EContentType::Custom));
- }
-
- void HandleWhileWorking(TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
- const auto& rec = ev->Get()->Record;
-
- if (UpdateFromDistributableConfig(rec.GetConfig().GetNetClassifierDistributableConfig())) {
- ReportGoodConfig();
- UpdateNetDataSource(ENetDataSourceType::DistributableConfig);
-
- BroadcastClassifierUpdate();
- } else {
- ReportBadConfig();
- }
-
- // report success
- auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec);
- Send(ev->Sender, resp.Release(), 0, ev->Cookie);
- }
-
- void HandleWhileWorking(TEvNetClassifier::TEvSubscribe::TPtr& ev) {
- AddSubscriber(ev->Sender);
-
- // respond instantly with the current classifier
- SendClassifierUpdate(ev->Sender);
- }
-
+ Subscribers.insert(subscriberId);
+
+ *Counters->SubscribersCount = Subscribers.size();
+ }
+
+ STATEFN(WaitingForSubscribers) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWaiting);
+ }
+ }
+
+ void SubscribeForConfigUpdates() {
+ const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem);
+ Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(kind));
+ }
+
+ void HandleWhileWaiting(TEvNetClassifier::TEvSubscribe::TPtr& ev) {
+ Become(&TThis::Initing);
+
+ InitCounters();
+
+ AddSubscriber(ev->Sender);
+
+ const auto& netDataFilePath = Cfg().GetNetDataFilePath();
+ if (netDataFilePath) {
+ MaybeNetDataFilePath = netDataFilePath;
+
+ MaybeNetDataFileModTs = GetNetDataFileModTimestamp(*MaybeNetDataFilePath);
+ }
+
+ SubscribeForConfigUpdates();
+ }
+
+ STATEFN(Initing) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileIniting);
+ hFunc(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleWhileIniting);
+ hFunc(TEvConfigsDispatcher::TEvGetConfigResponse, HandleWhileIniting);
+ hFunc(TEvents::TEvWakeup, HandleWhileIniting);
+ hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters);
+ }
+ }
+
+ void HandleWhileIniting(TEvNetClassifier::TEvSubscribe::TPtr& ev) {
+ AddSubscriber(ev->Sender);
+
+ // all subsribers get classifier update during a transition to a Working state
+ }
+
+ void RequestDistributableConfig() {
+ const auto kind = static_cast<ui32>(NKikimrConsole::TConfigItem::NetClassifierDistributableConfigItem);
+ Send(MakeConfigsDispatcherID(Ctx().SelfID.NodeId()), new TEvConfigsDispatcher::TEvGetConfigRequest(kind));
+ }
+
+ void HandleWhileIniting(TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) {
+ LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "subscribed to ConfigsDispatcher for NetData updates");
+
+ // Subscription is set but the console tablet may still be unavailable hence it's a good idea to schedule a timeout event
+ Schedule(TDuration::Seconds(Cfg().GetCmsConfigTimeoutSeconds()), new TEvents::TEvWakeup);
+
+ RequestDistributableConfig();
+
+ // This leads to Case 1 or Case 2
+ }
+
+ // Case 1: GetConfig request timed out
+ void HandleWhileIniting(TEvents::TEvWakeup::TPtr&) {
+ LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "TIMEOUT: failed to get distributable config, fall back to file: " << MaybeNetDataFilePath);
+ InitFromFile();
+ }
+
+ template<typename TConfPtr>
+ bool ShouldUseDistributableConfigOnInit(const TConfPtr& ptr) const {
+ if (!ptr) {
+ return false;
+ }
+
+ if (!ptr->HasNetClassifierDistributableConfig()) {
+ return false;
+ }
+
+ const auto& config = ptr->GetNetClassifierDistributableConfig();
+ if (config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp()) {
+ if (MaybeNetDataFileModTs) {
+ return TInstant::MicroSeconds(config.GetLastUpdateTimestamp()) > *MaybeNetDataFileModTs;
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
+ // Case 2: GetConfig request succeeded
+ void HandleWhileIniting(TEvConfigsDispatcher::TEvGetConfigResponse::TPtr& ev) {
+ if (ShouldUseDistributableConfigOnInit(ev->Get()->Config)) {
+ LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will initialize from distributable config");
+
+ if (UpdateFromDistributableConfig(ev->Get()->Config->GetNetClassifierDistributableConfig())) {
+ ReportGoodConfig();
+ UpdateNetDataSource(ENetDataSourceType::DistributableConfig);
+
+ LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from distributable config");
+
+ CompleteInitialization();
+ return;
+ } else {
+ LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from distributed config");
+ }
+ }
+
+ ReportBadConfig();
+ LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "distributable config is empty, broken or outdated, will use file: " << MaybeNetDataFilePath);
+ InitFromFile();
+ }
+
+ void InitFromFile() {
+ // NetData file may be outdated hence warning log level is used
+ LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "will try to initialize from file: " << MaybeNetDataFilePath);
+ LabeledAddressClassifier = TryReadNetDataFromFile();
+ if (LabeledAddressClassifier) {
+ NetDataUpdateTimestamp = MaybeNetDataFileModTs;
+ UpdateNetDataSource(ENetDataSourceType::File);
+
+ LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "successfully initialized from file: " << MaybeNetDataFilePath);
+ } else {
+ LOG_WARN_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to initialize from file: " << MaybeNetDataFilePath);
+ }
+
+ CompleteInitialization();
+ }
+
+ bool CheckDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) const {
+ return config.GetPackedNetData() && config.GetLastUpdateDatetimeUTC() && config.GetLastUpdateTimestamp();
+ }
+
+ bool UpdateFromDistributableConfig(const NKikimrNetClassifier::TNetClassifierDistributableConfig& config) {
+ if (CheckDistributableConfig(config)) {
+ LOG_INFO_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got new config with datetime: " << config.GetLastUpdateDatetimeUTC());
+
+ auto labeledAddressClassifier = BuildNetClassifierFromPackedNetData(config.GetPackedNetData());
+ if (labeledAddressClassifier) {
+ LabeledAddressClassifier = std::move(labeledAddressClassifier);
+ NetDataUpdateTimestamp = TInstant::MicroSeconds(config.GetLastUpdateTimestamp());
+
+ return true;
+ } else {
+ LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to parse NetData from distributable configuration");
+ }
+ } else {
+ LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "got bad distributable configuration");
+ }
+
+ return false;
+ }
+
+ void CompleteInitialization() {
+ Become(&TThis::Working);
+
+ NActors::TMon* mon = AppData(Ctx())->Mon;
+ if (mon) {
+ NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors");
+ mon->RegisterActorPage(page, "netclassifier", "NetClassifier", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID);
+ }
+
+ BroadcastClassifierUpdate();
+ }
+
+ STATEFN(Working) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvConsole::TEvConfigNotificationRequest, HandleWhileWorking);
+ hFunc(TEvNetClassifier::TEvSubscribe, HandleWhileWorking);
+ hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
+ hFunc(TEvNetClassifier::TEvUpdateTimedCounters, UpdateTimedCounters);
+ }
+ }
+
+ static TString FormatTs(const TMaybe<TInstant>& time) {
+ if (time) {
+ return time->ToRfc822String();
+ }
+ return "NULL";
+ }
+
+ void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev)
+ {
+ TStringBuilder message;
+ message << "NetClassifier info: \n";
+ message << "\tUnique subscribers: " << Subscribers.size() << "\n";
+ message << "\tClassifier updates sent: " << ClassifierUpdatesSent << "\n";
+ message << "\tLast classifier update broadcast ts: " << FormatTs(LastClassifierUpdateBroadcastTs) << "\n";
+ message << "\tNetData file path: " << MaybeNetDataFilePath << "\n";
+ message << "\tNetData file ts: " << FormatTs(MaybeNetDataFileModTs) << "\n";
+ message << "\tCurrent NetData ts: " << FormatTs(NetDataUpdateTimestamp) << "\n";
+ message << "\tGood config notifications: " << GoodConfigNotifications << "\n";
+ message << "\tBroken or empty config notifications: " << BrokenOrEmptyConfigNotifications << "\n";
+ message << "\tCurrent NetData source: " << ToString(CurrentNetDataSource) << "\n";
+ message << "\tClassifier: " << (LabeledAddressClassifier ? "OK" : "NULL");
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(NMonitoring::HTTPOKTEXT) + TString(message), 0,
+ NMon::IEvHttpInfoRes::EContentType::Custom));
+ }
+
+ void HandleWhileWorking(TEvConsole::TEvConfigNotificationRequest::TPtr& ev) {
+ const auto& rec = ev->Get()->Record;
+
+ if (UpdateFromDistributableConfig(rec.GetConfig().GetNetClassifierDistributableConfig())) {
+ ReportGoodConfig();
+ UpdateNetDataSource(ENetDataSourceType::DistributableConfig);
+
+ BroadcastClassifierUpdate();
+ } else {
+ ReportBadConfig();
+ }
+
+ // report success
+ auto resp = MakeHolder<TEvConsole::TEvConfigNotificationResponse>(rec);
+ Send(ev->Sender, resp.Release(), 0, ev->Cookie);
+ }
+
+ void HandleWhileWorking(TEvNetClassifier::TEvSubscribe::TPtr& ev) {
+ AddSubscriber(ev->Sender);
+
+ // respond instantly with the current classifier
+ SendClassifierUpdate(ev->Sender);
+ }
+
void SendClassifierUpdate(const TActorId recipient) {
- auto ev = MakeHolder<TEvNetClassifier::TEvClassifierUpdate>();
- ev->Classifier = LabeledAddressClassifier;
- ev->NetDataUpdateTimestamp = NetDataUpdateTimestamp;
-
- Send(recipient, ev.Release());
-
- ++ClassifierUpdatesSent;
- }
-
- void BroadcastClassifierUpdate() {
- for (const auto& subscriberId : Subscribers) {
- SendClassifierUpdate(subscriberId);
- }
-
- LastClassifierUpdateBroadcastTs = Ctx().Now();
- }
-
- TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromPackedNetData(const TString& packedNetData) const {
- const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(packedNetData);
- if (!serializedNetData) {
- LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "empty packed networks");
- return nullptr;
- }
-
- NKikimrNetClassifier::TNetData netData;
- if (!netData.ParseFromString(serializedNetData)) {
- LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "can't deserialize networks data protobuf");
- return nullptr;
- }
-
- return BuildNetClassifierFromNetData(netData);
- }
-
- TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromNetData(const NKikimrNetClassifier::TNetData& netData) const {
- auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData);
- if (!labeledAddressClassifier) {
- LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "invalid NetData format");
- }
-
- return labeledAddressClassifier;
- }
-
- TLabeledAddressClassifier::TConstPtr TryReadNetDataFromFile() const {
- if (MaybeNetDataFilePath) {
- try {
- return ReadNetDataFromFile(*MaybeNetDataFilePath);
- } catch (const yexception& ex) {
- LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to read NetData from file: " << ex.what());
- }
- }
-
- return nullptr;
- }
-
- TLabeledAddressClassifier::TConstPtr ReadNetDataFromFile(const TString& path) const {
- NKikimrNetClassifier::TNetData netData;
-
- // Parse TSV-format
- TFileInput knownNetsFile(path);
- TString line;
- while (knownNetsFile.ReadLine(line)) {
- TStringBuf mask, label;
- if (!TStringBuf(line).TryRSplit('\t', mask, label)) {
- ythrow yexception() << "NetData file violates tsv format";
- }
-
- auto& subnet = *netData.AddSubnets();
- subnet.SetMask(TString(mask));
- subnet.SetLabel(TString(label));
- }
-
- return BuildNetClassifierFromNetData(netData);
- }
-
- // Monitoring features
-
- void ReportGoodConfig() {
- ++GoodConfigNotifications;
- Counters->GoodConfigNotificationsCount->Inc();
- }
-
- void ReportBadConfig() {
- ++BrokenOrEmptyConfigNotifications;
- Counters->BrokenConfigNotificationsCount->Inc();
- }
-
- void UpdateNetDataSource(const ENetDataSourceType newSource) {
- CurrentNetDataSource = newSource;
- *Counters->NetDataSourceType = CurrentNetDataSource;
- }
-
- void UpdateTimedCounters(TEvNetClassifier::TEvUpdateTimedCounters::TPtr&) {
- if (NetDataUpdateTimestamp) {
- (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTimestamp->Seconds();
- }
-
- Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvNetClassifier::TEvUpdateTimedCounters);
- }
-
- void InitCounters() {
- Counters = MakeIntrusive<TNetClassifierCounters>(GetServiceCounters(AppData(Ctx())->Counters, "netclassifier")->GetSubgroup("subsystem", "distributor"));
-
- Send(Ctx().SelfID, new TEvNetClassifier::TEvUpdateTimedCounters);
- }
-
-private:
- TMaybe<TString> MaybeNetDataFilePath;
- TMaybe<TInstant> MaybeNetDataFileModTs;
-
- TLabeledAddressClassifier::TConstPtr LabeledAddressClassifier;
- TMaybe<TInstant> NetDataUpdateTimestamp;
+ auto ev = MakeHolder<TEvNetClassifier::TEvClassifierUpdate>();
+ ev->Classifier = LabeledAddressClassifier;
+ ev->NetDataUpdateTimestamp = NetDataUpdateTimestamp;
+
+ Send(recipient, ev.Release());
+
+ ++ClassifierUpdatesSent;
+ }
+
+ void BroadcastClassifierUpdate() {
+ for (const auto& subscriberId : Subscribers) {
+ SendClassifierUpdate(subscriberId);
+ }
+
+ LastClassifierUpdateBroadcastTs = Ctx().Now();
+ }
+
+ TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromPackedNetData(const TString& packedNetData) const {
+ const TString serializedNetData = NNetClassifierUpdater::UnpackNetData(packedNetData);
+ if (!serializedNetData) {
+ LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "empty packed networks");
+ return nullptr;
+ }
+
+ NKikimrNetClassifier::TNetData netData;
+ if (!netData.ParseFromString(serializedNetData)) {
+ LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "can't deserialize networks data protobuf");
+ return nullptr;
+ }
+
+ return BuildNetClassifierFromNetData(netData);
+ }
+
+ TLabeledAddressClassifier::TConstPtr BuildNetClassifierFromNetData(const NKikimrNetClassifier::TNetData& netData) const {
+ auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData);
+ if (!labeledAddressClassifier) {
+ LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "invalid NetData format");
+ }
+
+ return labeledAddressClassifier;
+ }
+
+ TLabeledAddressClassifier::TConstPtr TryReadNetDataFromFile() const {
+ if (MaybeNetDataFilePath) {
+ try {
+ return ReadNetDataFromFile(*MaybeNetDataFilePath);
+ } catch (const yexception& ex) {
+ LOG_ERROR_S(Ctx(), NKikimrServices::NET_CLASSIFIER, "failed to read NetData from file: " << ex.what());
+ }
+ }
+
+ return nullptr;
+ }
+
+ TLabeledAddressClassifier::TConstPtr ReadNetDataFromFile(const TString& path) const {
+ NKikimrNetClassifier::TNetData netData;
+
+ // Parse TSV-format
+ TFileInput knownNetsFile(path);
+ TString line;
+ while (knownNetsFile.ReadLine(line)) {
+ TStringBuf mask, label;
+ if (!TStringBuf(line).TryRSplit('\t', mask, label)) {
+ ythrow yexception() << "NetData file violates tsv format";
+ }
+
+ auto& subnet = *netData.AddSubnets();
+ subnet.SetMask(TString(mask));
+ subnet.SetLabel(TString(label));
+ }
+
+ return BuildNetClassifierFromNetData(netData);
+ }
+
+ // Monitoring features
+
+ void ReportGoodConfig() {
+ ++GoodConfigNotifications;
+ Counters->GoodConfigNotificationsCount->Inc();
+ }
+
+ void ReportBadConfig() {
+ ++BrokenOrEmptyConfigNotifications;
+ Counters->BrokenConfigNotificationsCount->Inc();
+ }
+
+ void UpdateNetDataSource(const ENetDataSourceType newSource) {
+ CurrentNetDataSource = newSource;
+ *Counters->NetDataSourceType = CurrentNetDataSource;
+ }
+
+ void UpdateTimedCounters(TEvNetClassifier::TEvUpdateTimedCounters::TPtr&) {
+ if (NetDataUpdateTimestamp) {
+ (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTimestamp->Seconds();
+ }
+
+ Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvNetClassifier::TEvUpdateTimedCounters);
+ }
+
+ void InitCounters() {
+ Counters = MakeIntrusive<TNetClassifierCounters>(GetServiceCounters(AppData(Ctx())->Counters, "netclassifier")->GetSubgroup("subsystem", "distributor"));
+
+ Send(Ctx().SelfID, new TEvNetClassifier::TEvUpdateTimedCounters);
+ }
+
+private:
+ TMaybe<TString> MaybeNetDataFilePath;
+ TMaybe<TInstant> MaybeNetDataFileModTs;
+
+ TLabeledAddressClassifier::TConstPtr LabeledAddressClassifier;
+ TMaybe<TInstant> NetDataUpdateTimestamp;
THashSet<TActorId> Subscribers;
-
- ui64 ClassifierUpdatesSent = 0;
- ui64 GoodConfigNotifications = 0;
- ui64 BrokenOrEmptyConfigNotifications = 0;
- TMaybe<TInstant> LastClassifierUpdateBroadcastTs;
- ENetDataSourceType CurrentNetDataSource = ENetDataSourceType::None;
-
- TNetClassifierCounters::TPtr Counters;
-};
-
-NActors::IActor* CreateNetClassifier() {
- return new TNetClassifier();
-}
-
-} // namespace NKikimr::NNetClassifier
+
+ ui64 ClassifierUpdatesSent = 0;
+ ui64 GoodConfigNotifications = 0;
+ ui64 BrokenOrEmptyConfigNotifications = 0;
+ TMaybe<TInstant> LastClassifierUpdateBroadcastTs;
+ ENetDataSourceType CurrentNetDataSource = ENetDataSourceType::None;
+
+ TNetClassifierCounters::TPtr Counters;
+};
+
+NActors::IActor* CreateNetClassifier() {
+ return new TNetClassifier();
+}
+
+} // namespace NKikimr::NNetClassifier
diff --git a/ydb/core/mind/address_classification/net_classifier.h b/ydb/core/mind/address_classification/net_classifier.h
index 4bc4f3b6a7..6a298cd4c4 100644
--- a/ydb/core/mind/address_classification/net_classifier.h
+++ b/ydb/core/mind/address_classification/net_classifier.h
@@ -1,56 +1,56 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/base/events.h>
#include <ydb/core/util/address_classifier.h>
-
+
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/actors/core/actorid.h>
#include <library/cpp/actors/core/defs.h>
#include <library/cpp/actors/core/event_local.h>
-
+
#include <library/cpp/monlib/dynamic_counters/counters.h>
-
-#include <util/generic/ptr.h>
-
-#include <vector>
-
-namespace NKikimr::NNetClassifier {
-
+
+#include <util/generic/ptr.h>
+
+#include <vector>
+
+namespace NKikimr::NNetClassifier {
+
using NActors::TActorId;
-
+
inline TActorId MakeNetClassifierID() {
- static const char x[12] = "net_classvc";
+ static const char x[12] = "net_classvc";
return TActorId(0, TStringBuf(x, 12));
-}
-
-NActors::IActor* CreateNetClassifier();
-
-struct TEvNetClassifier {
- enum EEv {
- EvClassifierUpdate = EventSpaceBegin(TKikimrEvents::ES_NET_CLASSIFIER),
- EvSubscribe,
- EvUpdateTimedCounters,
- EvEnd
- };
-
- struct TEvClassifierUpdate : public NActors::TEventLocal<TEvClassifierUpdate, EvClassifierUpdate> {
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr Classifier;
- TMaybe<TInstant> NetDataUpdateTimestamp;
- };
-
- struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> {
- };
-
- struct TEvUpdateTimedCounters : public NActors::TEventLocal<TEvUpdateTimedCounters, EvUpdateTimedCounters> {
- };
-
- static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_NET_CLASSIFIER), "Unexpected TEvNetClassifier event range");
-};
-
-enum ENetDataSourceType {
- None = 0,
- File,
- DistributableConfig
-};
-
-} // namespace NKikimr::NNetClassifier
+}
+
+NActors::IActor* CreateNetClassifier();
+
+struct TEvNetClassifier {
+ enum EEv {
+ EvClassifierUpdate = EventSpaceBegin(TKikimrEvents::ES_NET_CLASSIFIER),
+ EvSubscribe,
+ EvUpdateTimedCounters,
+ EvEnd
+ };
+
+ struct TEvClassifierUpdate : public NActors::TEventLocal<TEvClassifierUpdate, EvClassifierUpdate> {
+ NAddressClassifier::TLabeledAddressClassifier::TConstPtr Classifier;
+ TMaybe<TInstant> NetDataUpdateTimestamp;
+ };
+
+ struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> {
+ };
+
+ struct TEvUpdateTimedCounters : public NActors::TEventLocal<TEvUpdateTimedCounters, EvUpdateTimedCounters> {
+ };
+
+ static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_NET_CLASSIFIER), "Unexpected TEvNetClassifier event range");
+};
+
+enum ENetDataSourceType {
+ None = 0,
+ File,
+ DistributableConfig
+};
+
+} // namespace NKikimr::NNetClassifier
diff --git a/ydb/core/mind/address_classification/net_classifier_ut.cpp b/ydb/core/mind/address_classification/net_classifier_ut.cpp
index 7ba36c87c4..088d25b9c7 100644
--- a/ydb/core/mind/address_classification/net_classifier_ut.cpp
+++ b/ydb/core/mind/address_classification/net_classifier_ut.cpp
@@ -1,219 +1,219 @@
#include <ydb/core/base/counters.h>
-
+
#include <ydb/core/mind/address_classification/counters.h>
-
+
#include <ydb/core/mind/address_classification/net_classifier.h>
-
+
#include <ydb/core/testlib/test_client.h>
-
+
#include <library/cpp/actors/http/http_proxy.cpp>
#include <library/cpp/testing/unittest/tests_data.h>
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/stream/file.h>
-#include <util/system/tempfile.h>
-
-namespace NKikimr::NNetClassifierTests {
-
-using namespace NNetClassifier;
-using namespace Tests;
-
-static THolder<TTempFileHandle> CreateNetDataFile(const TString& content) {
- auto netDataFile = MakeHolder<TTempFileHandle>("data.tsv");
-
- netDataFile->Write(content.Data(), content.Size());
- netDataFile->FlushData();
-
- return netDataFile;
-}
-
-static void Wait() {
- Sleep(TDuration::MilliSeconds(333));
-}
-
-static TString FormNetData() {
- return "10.99.99.224/27\tSAS\n"
- "185.32.186.0/29\tIVA\n"
- "185.32.186.16/29\tMYT\n"
- "185.32.186.24/31\tIVA\n"
- "185.32.186.26/31\tIVA\n"
- "37.29.21.132/30\tMYT\n"
- "83.220.51.8/30\tMYT\n"
- "2a02:6b8:f030:3000::/52\tMYT\n"
- "2a02:6b8:fc00::/48\tSAS";
-}
-
+
+#include <util/stream/file.h>
+#include <util/system/tempfile.h>
+
+namespace NKikimr::NNetClassifierTests {
+
+using namespace NNetClassifier;
+using namespace Tests;
+
+static THolder<TTempFileHandle> CreateNetDataFile(const TString& content) {
+ auto netDataFile = MakeHolder<TTempFileHandle>("data.tsv");
+
+ netDataFile->Write(content.Data(), content.Size());
+ netDataFile->FlushData();
+
+ return netDataFile;
+}
+
+static void Wait() {
+ Sleep(TDuration::MilliSeconds(333));
+}
+
+static TString FormNetData() {
+ return "10.99.99.224/27\tSAS\n"
+ "185.32.186.0/29\tIVA\n"
+ "185.32.186.16/29\tMYT\n"
+ "185.32.186.24/31\tIVA\n"
+ "185.32.186.26/31\tIVA\n"
+ "37.29.21.132/30\tMYT\n"
+ "83.220.51.8/30\tMYT\n"
+ "2a02:6b8:f030:3000::/52\tMYT\n"
+ "2a02:6b8:fc00::/48\tSAS";
+}
+
static TAutoPtr<IEventHandle> GetClassifierUpdate(TServer& server, const TActorId sender) {
- auto& actorSystem = *server.GetRuntime();
-
- actorSystem.Send(
- new IEventHandle(MakeNetClassifierID(), sender,
- new TEvNetClassifier::TEvSubscribe()
- ));
-
- TAutoPtr<IEventHandle> handle;
- actorSystem.GrabEdgeEvent<TEvNetClassifier::TEvClassifierUpdate>(handle);
-
- UNIT_ASSERT(handle);
- UNIT_ASSERT_VALUES_EQUAL(handle->Recipient, sender);
-
- return handle;
-}
-
-static NCounters::TNetClassifierCounters::TPtr ExtractCounters(TServer& server) {
- UNIT_ASSERT_VALUES_EQUAL(server.GetRuntime()->GetNodeCount(), 1);
-
- return MakeIntrusive<NCounters::TNetClassifierCounters>(
- GetServiceCounters(server.GetRuntime()->GetAppData(0).Counters, "netclassifier")->GetSubgroup("subsystem", "distributor")
- );
-}
-
-Y_UNIT_TEST_SUITE(TNetClassifierTest) {
- Y_UNIT_TEST(TestInitFromFile) {
- TPortManager pm;
- const ui16 port = pm.GetPort(2134);
-
- TServerSettings settings(port);
-
- auto netDataFile = CreateNetDataFile(FormNetData());
- settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name());
- settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1);
-
- TServer cleverServer = TServer(settings);
- {
+ auto& actorSystem = *server.GetRuntime();
+
+ actorSystem.Send(
+ new IEventHandle(MakeNetClassifierID(), sender,
+ new TEvNetClassifier::TEvSubscribe()
+ ));
+
+ TAutoPtr<IEventHandle> handle;
+ actorSystem.GrabEdgeEvent<TEvNetClassifier::TEvClassifierUpdate>(handle);
+
+ UNIT_ASSERT(handle);
+ UNIT_ASSERT_VALUES_EQUAL(handle->Recipient, sender);
+
+ return handle;
+}
+
+static NCounters::TNetClassifierCounters::TPtr ExtractCounters(TServer& server) {
+ UNIT_ASSERT_VALUES_EQUAL(server.GetRuntime()->GetNodeCount(), 1);
+
+ return MakeIntrusive<NCounters::TNetClassifierCounters>(
+ GetServiceCounters(server.GetRuntime()->GetAppData(0).Counters, "netclassifier")->GetSubgroup("subsystem", "distributor")
+ );
+}
+
+Y_UNIT_TEST_SUITE(TNetClassifierTest) {
+ Y_UNIT_TEST(TestInitFromFile) {
+ TPortManager pm;
+ const ui16 port = pm.GetPort(2134);
+
+ TServerSettings settings(port);
+
+ auto netDataFile = CreateNetDataFile(FormNetData());
+ settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name());
+ settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1);
+
+ TServer cleverServer = TServer(settings);
+ {
const TActorId sender = cleverServer.GetRuntime()->AllocateEdgeActor();
- auto handle = GetClassifierUpdate(cleverServer, sender);
- auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>();
-
- UNIT_ASSERT(event->NetDataUpdateTimestamp);
- UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero());
-
- // basic sanity check
- UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("37.29.21.135"), "MYT");
- UNIT_ASSERT(!event->Classifier->ClassifyAddress("trololo"));
- }
- {
- auto counters = ExtractCounters(cleverServer);
-
- UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1);
- UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0);
- UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1);
- UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::File);
-
+ auto handle = GetClassifierUpdate(cleverServer, sender);
+ auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>();
+
+ UNIT_ASSERT(event->NetDataUpdateTimestamp);
+ UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero());
+
+ // basic sanity check
+ UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("37.29.21.135"), "MYT");
+ UNIT_ASSERT(!event->Classifier->ClassifyAddress("trololo"));
+ }
+ {
+ auto counters = ExtractCounters(cleverServer);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::File);
+
const auto prevLagSeconds = AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic());
while (prevLagSeconds >= AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic())) {
- Wait();
- }
- }
- }
-
- Y_UNIT_TEST(TestInitFromBadlyFormattedFile) {
- TPortManager pm;
- const ui16 port = pm.GetPort(2134);
-
- TServerSettings settings(port);
-
- auto netDataFile = CreateNetDataFile("omg\tlol");
- settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name());
-
- TServer cleverServer = TServer(settings);
- {
+ Wait();
+ }
+ }
+ }
+
+ Y_UNIT_TEST(TestInitFromBadlyFormattedFile) {
+ TPortManager pm;
+ const ui16 port = pm.GetPort(2134);
+
+ TServerSettings settings(port);
+
+ auto netDataFile = CreateNetDataFile("omg\tlol");
+ settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name());
+
+ TServer cleverServer = TServer(settings);
+ {
const TActorId sender = cleverServer.GetRuntime()->AllocateEdgeActor();
- auto handle = GetClassifierUpdate(cleverServer, sender);
- auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>();
-
- UNIT_ASSERT(!event->NetDataUpdateTimestamp);
- UNIT_ASSERT(!event->Classifier);
- }
- {
- auto counters = ExtractCounters(cleverServer);
-
- UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1);
- UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0);
- UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1);
- UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::None);
- }
- }
-
- static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) {
- const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: "
- << netData.size() << "\r\n\r\n" << netData;
- NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content);
-
- return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse);
- }
-
- Y_UNIT_TEST(TestInitFromRemoteSource) {
+ auto handle = GetClassifierUpdate(cleverServer, sender);
+ auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>();
+
+ UNIT_ASSERT(!event->NetDataUpdateTimestamp);
+ UNIT_ASSERT(!event->Classifier);
+ }
+ {
+ auto counters = ExtractCounters(cleverServer);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters->SubscribersCount->GetAtomic(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(counters->GoodConfigNotificationsCount->GetAtomic(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(counters->BrokenConfigNotificationsCount->GetAtomic(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(counters->NetDataSourceType->GetAtomic(), ENetDataSourceType::None);
+ }
+ }
+
+ static NHttp::TEvHttpProxy::TEvHttpOutgoingResponse* MakeHttpResponse(NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request, const TString& netData) {
+ const TString content = TStringBuilder() << "HTTP/1.1 200 OK\r\nConnection: Close\r\nContent-Type: application/octet-stream\r\nContent-Length: "
+ << netData.size() << "\r\n\r\n" << netData;
+ NHttp::THttpOutgoingResponsePtr httpResponse = request->Request->CreateResponseString(content);
+
+ return new NHttp::TEvHttpProxy::TEvHttpOutgoingResponse(httpResponse);
+ }
+
+ Y_UNIT_TEST(TestInitFromRemoteSource) {
NMonitoring::TMetricRegistry sensors;
-
- TPortManager pm;
- const ui16 port = pm.GetPort(2134);
- const ui64 netDataSourcePort = pm.GetPort(13334);
-
- const TString hostAndPort = TStringBuilder() << "http://[::1]:" << netDataSourcePort;
- const TString uri = "/fancy_path/networks.tsv";
-
- TServerSettings settings(port);
- auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig();
- updaterConfig.SetNetDataSourceUrl(hostAndPort + uri);
- updaterConfig.SetNetDataUpdateIntervalSeconds(1);
- updaterConfig.SetRetryIntervalSeconds(1);
-
- settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1);
-
- TServer cleverServer = TServer(settings);
- auto& actorSystem = *cleverServer.GetRuntime();
-
- NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors);
+
+ TPortManager pm;
+ const ui16 port = pm.GetPort(2134);
+ const ui64 netDataSourcePort = pm.GetPort(13334);
+
+ const TString hostAndPort = TStringBuilder() << "http://[::1]:" << netDataSourcePort;
+ const TString uri = "/fancy_path/networks.tsv";
+
+ TServerSettings settings(port);
+ auto& updaterConfig = *settings.NetClassifierConfig.MutableUpdaterConfig();
+ updaterConfig.SetNetDataSourceUrl(hostAndPort + uri);
+ updaterConfig.SetNetDataUpdateIntervalSeconds(1);
+ updaterConfig.SetRetryIntervalSeconds(1);
+
+ settings.NetClassifierConfig.SetTimedCountersUpdateIntervalSeconds(1);
+
+ TServer cleverServer = TServer(settings);
+ auto& actorSystem = *cleverServer.GetRuntime();
+
+ NActors::IActor* proxy = NHttp::CreateHttpProxy(sensors);
NActors::TActorId proxyId = actorSystem.Register(proxy);
-
- actorSystem.Send(
+
+ actorSystem.Send(
new NActors::IEventHandle(proxyId, TActorId(), new NHttp::TEvHttpProxy::TEvAddListeningPort(netDataSourcePort)), 0, true
- );
-
+ );
+
NActors::TActorId serverId = actorSystem.AllocateEdgeActor();
- actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(uri, serverId)), 0, true);
-
+ actorSystem.Send(new NActors::IEventHandle(proxyId, serverId, new NHttp::TEvHttpProxy::TEvRegisterHandler(uri, serverId)), 0, true);
+
const TActorId sender = cleverServer.GetRuntime()->AllocateEdgeActor();
-
- // initially classifier's data should be empty
- auto handle = GetClassifierUpdate(cleverServer, sender);
- UNIT_ASSERT(!handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier);
-
- // then the server responds with new net data
- NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle);
- UNIT_ASSERT_EQUAL(request->Request->URL, uri);
-
- actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, FormNetData())), 0, true);
-
- // subscriber waits for proper net data update
- while (true) {
- handle = GetClassifierUpdate(cleverServer, sender);
-
- if (handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier) {
- break;
- }
-
- Wait();
- }
-
- UNIT_ASSERT(handle);
- auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>();
-
- UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("185.32.186.26"), "IVA");
-
- {
- auto counters = ExtractCounters(cleverServer);
-
+
+ // initially classifier's data should be empty
+ auto handle = GetClassifierUpdate(cleverServer, sender);
+ UNIT_ASSERT(!handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier);
+
+ // then the server responds with new net data
+ NHttp::TEvHttpProxy::TEvHttpIncomingRequest* request = actorSystem.GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingRequest>(handle);
+ UNIT_ASSERT_EQUAL(request->Request->URL, uri);
+
+ actorSystem.Send(new NActors::IEventHandle(handle->Sender, serverId, MakeHttpResponse(request, FormNetData())), 0, true);
+
+ // subscriber waits for proper net data update
+ while (true) {
+ handle = GetClassifierUpdate(cleverServer, sender);
+
+ if (handle->Get<TEvNetClassifier::TEvClassifierUpdate>()->Classifier) {
+ break;
+ }
+
+ Wait();
+ }
+
+ UNIT_ASSERT(handle);
+ auto* event = handle->Get<TEvNetClassifier::TEvClassifierUpdate>();
+
+ UNIT_ASSERT_STRINGS_EQUAL(*event->Classifier->ClassifyAddress("185.32.186.26"), "IVA");
+
+ {
+ auto counters = ExtractCounters(cleverServer);
+
UNIT_ASSERT_VALUES_EQUAL(AtomicGet(counters->SubscribersCount->GetAtomic()), 1);
UNIT_ASSERT_VALUES_EQUAL(AtomicGet(counters->NetDataSourceType->GetAtomic()), ENetDataSourceType::DistributableConfig);
-
+
const auto prevLagSeconds = AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic());
while (prevLagSeconds >= AtomicGet(counters->NetDataUpdateLagSeconds->GetAtomic())) {
- Wait();
- }
- }
- }
-}
-
-} // namespace NKikimr::NNetClassifierTests
+ Wait();
+ }
+ }
+ }
+}
+
+} // namespace NKikimr::NNetClassifierTests
diff --git a/ydb/core/mind/address_classification/ut/ya.make b/ydb/core/mind/address_classification/ut/ya.make
index b7cab35ea2..62157a5aff 100644
--- a/ydb/core/mind/address_classification/ut/ya.make
+++ b/ydb/core/mind/address_classification/ut/ya.make
@@ -1,34 +1,34 @@
UNITTEST_FOR(ydb/core/mind/address_classification)
-
+
OWNER(
radix
g:kikimr
)
-
-FORK_SUBTESTS()
-
-IF (SANITIZER_TYPE OR WITH_VALGRIND)
- TIMEOUT(1200)
- SIZE(LARGE)
- SPLIT_FACTOR(20)
- TAG(ya:fat)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ TIMEOUT(1200)
+ SIZE(LARGE)
+ SPLIT_FACTOR(20)
+ TAG(ya:fat)
REQUIREMENTS(ram:16)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
REQUIREMENTS(ram:16)
-ENDIF()
-
-PEERDIR(
+ENDIF()
+
+PEERDIR(
library/cpp/actors/http
ydb/core/mind/address_classification
ydb/core/testlib
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-SRCS(
- net_classifier_ut.cpp
-)
-
-END()
+SRCS(
+ net_classifier_ut.cpp
+)
+
+END()
diff --git a/ydb/core/mind/address_classification/ya.make b/ydb/core/mind/address_classification/ya.make
index 87f74b5b95..73301851da 100644
--- a/ydb/core/mind/address_classification/ya.make
+++ b/ydb/core/mind/address_classification/ya.make
@@ -1,18 +1,18 @@
-LIBRARY()
-
-OWNER(
- radix
- g:kikimr
-)
-
-SRCS(
- counters.cpp
- net_classifier.cpp
-)
-
-GENERATE_ENUM_SERIALIZATION(net_classifier.h)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ radix
+ g:kikimr
+)
+
+SRCS(
+ counters.cpp
+ net_classifier.cpp
+)
+
+GENERATE_ENUM_SERIALIZATION(net_classifier.h)
+
+PEERDIR(
library/cpp/actors/core
library/cpp/monlib/dynamic_counters
ydb/core/base
@@ -20,9 +20,9 @@ PEERDIR(
ydb/core/mon
ydb/core/protos
ydb/core/util
-)
-
-END()
+)
+
+END()
RECURSE_FOR_TESTS(
ut
diff --git a/ydb/core/persqueue/cluster_tracker.cpp b/ydb/core/persqueue/cluster_tracker.cpp
index a57e284f0e..59fdea5c2b 100644
--- a/ydb/core/persqueue/cluster_tracker.cpp
+++ b/ydb/core/persqueue/cluster_tracker.cpp
@@ -1,67 +1,67 @@
-#include "cluster_tracker.h"
+#include "cluster_tracker.h"
#include "pq_database.h"
-
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/kqp/kqp.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/scope.h>
-
-#include <tuple>
-#include <vector>
-
-namespace NKikimr::NPQ::NClusterTracker {
-
-inline auto& Ctx() {
- return TActivationContext::AsActorContext();
-}
-
-bool TClustersList::TCluster::operator==(const TCluster& rhs) const {
- return std::tie(Name, Datacenter, Balancer, IsLocal, IsEnabled, Weight) ==
- std::tie(rhs.Name, rhs.Datacenter, rhs.Balancer, rhs.IsLocal, rhs.IsEnabled, rhs.Weight);
-}
-
-bool TClustersList::operator==(const TClustersList& rhs) const {
- return Clusters == rhs.Clusters && Version == rhs.Version;
-}
-
-class TClusterTracker: public TActorBootstrapped<TClusterTracker> {
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_TRACKER;
- }
-
- const auto& Cfg() const {
- return AppData(Ctx())->PQConfig;
- }
-
- void Bootstrap() {
- ListClustersQuery = MakeListClustersQuery();
-
- Become(&TThis::WaitingForSubscribers);
- }
-
-private:
+
+#include <util/generic/hash.h>
+#include <util/generic/scope.h>
+
+#include <tuple>
+#include <vector>
+
+namespace NKikimr::NPQ::NClusterTracker {
+
+inline auto& Ctx() {
+ return TActivationContext::AsActorContext();
+}
+
+bool TClustersList::TCluster::operator==(const TCluster& rhs) const {
+ return std::tie(Name, Datacenter, Balancer, IsLocal, IsEnabled, Weight) ==
+ std::tie(rhs.Name, rhs.Datacenter, rhs.Balancer, rhs.IsLocal, rhs.IsEnabled, rhs.Weight);
+}
+
+bool TClustersList::operator==(const TClustersList& rhs) const {
+ return Clusters == rhs.Clusters && Version == rhs.Version;
+}
+
+class TClusterTracker: public TActorBootstrapped<TClusterTracker> {
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_TRACKER;
+ }
+
+ const auto& Cfg() const {
+ return AppData(Ctx())->PQConfig;
+ }
+
+ void Bootstrap() {
+ ListClustersQuery = MakeListClustersQuery();
+
+ Become(&TThis::WaitingForSubscribers);
+ }
+
+private:
void AddSubscriber(const TActorId subscriberId) {
- Subscribers.insert(subscriberId);
- }
-
- STATEFN(WaitingForSubscribers) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWaiting);
- }
- }
-
- void HandleWhileWaiting(TEvClusterTracker::TEvSubscribe::TPtr& ev) {
- AddSubscriber(ev->Sender);
-
+ Subscribers.insert(subscriberId);
+ }
+
+ STATEFN(WaitingForSubscribers) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWaiting);
+ }
+ }
+
+ void HandleWhileWaiting(TEvClusterTracker::TEvSubscribe::TPtr& ev) {
+ AddSubscriber(ev->Sender);
+
Become(&TThis::Working);
-
- Send(Ctx().SelfID, new TEvents::TEvWakeup);
- }
-
+
+ Send(Ctx().SelfID, new TEvents::TEvWakeup);
+ }
+
const TString& GetDatabase() {
if (Database.empty()) {
Database = GetDatabaseFromConfig(Cfg());
@@ -70,86 +70,86 @@ private:
return Database;
}
- TString MakeListClustersQuery() const {
- return Sprintf(
- R"(
+ TString MakeListClustersQuery() const {
+ return Sprintf(
+ R"(
--!syntax_v1
SELECT C.name, C.balancer, C.local, C.enabled, C.weight, V.version FROM `%s` AS C
- CROSS JOIN
+ CROSS JOIN
(SELECT version FROM `%s` WHERE name == 'Cluster') AS V;
- )", Cfg().GetClusterTablePath().c_str(), Cfg().GetVersionTablePath().c_str());
- }
-
- STATEFN(Working) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWorking);
- hFunc(TEvents::TEvWakeup, HandleWhileWorking);
- hFunc(NKqp::TEvKqp::TEvQueryResponse, HandleWhileWorking);
+ )", Cfg().GetClusterTablePath().c_str(), Cfg().GetVersionTablePath().c_str());
+ }
+
+ STATEFN(Working) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvClusterTracker::TEvSubscribe, HandleWhileWorking);
+ hFunc(TEvents::TEvWakeup, HandleWhileWorking);
+ hFunc(NKqp::TEvKqp::TEvQueryResponse, HandleWhileWorking);
hFunc(NKqp::TEvKqp::TEvProcessResponse, HandleWhileWorking);
- }
- }
-
+ }
+ }
+
void SendClustersList(TActorId subscriberId) {
- auto ev = MakeHolder<TEvClusterTracker::TEvClustersUpdate>();
-
- ev->ClustersList = ClustersList;
- ev->ClustersListUpdateTimestamp = ClustersListUpdateTimestamp;
-
- Send(subscriberId, ev.Release());
- }
-
- void HandleWhileWorking(TEvClusterTracker::TEvSubscribe::TPtr& ev) {
- AddSubscriber(ev->Sender);
-
- // List may be null due to reinit
- if (ClustersList) {
- SendClustersList(ev->Sender);
- }
- }
-
- void BroadcastClustersUpdate() {
- for (const auto& subscriberId : Subscribers) {
- SendClustersList(subscriberId);
- }
- }
-
- void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) {
- auto req = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
-
+ auto ev = MakeHolder<TEvClusterTracker::TEvClustersUpdate>();
+
+ ev->ClustersList = ClustersList;
+ ev->ClustersListUpdateTimestamp = ClustersListUpdateTimestamp;
+
+ Send(subscriberId, ev.Release());
+ }
+
+ void HandleWhileWorking(TEvClusterTracker::TEvSubscribe::TPtr& ev) {
+ AddSubscriber(ev->Sender);
+
+ // List may be null due to reinit
+ if (ClustersList) {
+ SendClustersList(ev->Sender);
+ }
+ }
+
+ void BroadcastClustersUpdate() {
+ for (const auto& subscriberId : Subscribers) {
+ SendClustersList(subscriberId);
+ }
+ }
+
+ void HandleWhileWorking(TEvents::TEvWakeup::TPtr&) {
+ auto req = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
+
req->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
req->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
- req->Record.MutableRequest()->SetKeepSession(false);
+ req->Record.MutableRequest()->SetKeepSession(false);
req->Record.MutableRequest()->SetQuery(MakeListClustersQuery());
req->Record.MutableRequest()->SetDatabase(GetDatabase());
// useless without explicit session
// req->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true);
req->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write();
req->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
-
- Send(NKqp::MakeKqpProxyID(Ctx().SelfID.NodeId()), req.Release());
- }
-
- void HandleWhileWorking(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev) {
- const auto& record = ev->Get()->Record.GetRef();
- if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS && record.GetResponse().GetResults(0).GetValue().GetStruct(0).ListSize()) {
- UpdateClustersList(record);
-
- Y_VERIFY(ClustersList);
- Y_VERIFY(ClustersList->Clusters.size());
- Y_VERIFY(ClustersListUpdateTimestamp && *ClustersListUpdateTimestamp);
-
- BroadcastClustersUpdate();
-
- Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutSec()), new TEvents::TEvWakeup);
- } else {
- LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record);
-
+
+ Send(NKqp::MakeKqpProxyID(Ctx().SelfID.NodeId()), req.Release());
+ }
+
+ void HandleWhileWorking(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record.GetRef();
+ if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS && record.GetResponse().GetResults(0).GetValue().GetStruct(0).ListSize()) {
+ UpdateClustersList(record);
+
+ Y_VERIFY(ClustersList);
+ Y_VERIFY(ClustersList->Clusters.size());
+ Y_VERIFY(ClustersListUpdateTimestamp && *ClustersListUpdateTimestamp);
+
+ BroadcastClustersUpdate();
+
+ Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutSec()), new TEvents::TEvWakeup);
+ } else {
+ LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record);
+
ClustersList = nullptr;
Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup);
- }
- }
-
+ }
+ }
+
void HandleWhileWorking(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev) {
const auto& record = ev->Get()->Record;
LOG_ERROR_S(Ctx(), NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, "failed to list clusters: " << record);
@@ -159,41 +159,41 @@ private:
Schedule(TDuration::Seconds(Cfg().GetClustersUpdateTimeoutOnErrorSec()), new TEvents::TEvWakeup);
}
- template<typename TProtoRecord>
- void UpdateClustersList(const TProtoRecord& record) {
- auto clustersList = MakeIntrusive<TClustersList>();
- auto& t = record.GetResponse().GetResults(0).GetValue().GetStruct(0);
- clustersList->Clusters.resize(t.ListSize());
-
- for (size_t i = 0; i < t.ListSize(); ++i) {
- auto& cluster = clustersList->Clusters[i];
-
- cluster.Name = t.GetList(i).GetStruct(0).GetOptional().GetText();
- cluster.Datacenter = cluster.Name;
- cluster.Balancer = t.GetList(i).GetStruct(1).GetOptional().GetText();
-
- cluster.IsLocal = t.GetList(i).GetStruct(2).GetOptional().GetBool();
- cluster.IsEnabled = t.GetList(i).GetStruct(3).GetOptional().GetBool();
+ template<typename TProtoRecord>
+ void UpdateClustersList(const TProtoRecord& record) {
+ auto clustersList = MakeIntrusive<TClustersList>();
+ auto& t = record.GetResponse().GetResults(0).GetValue().GetStruct(0);
+ clustersList->Clusters.resize(t.ListSize());
+
+ for (size_t i = 0; i < t.ListSize(); ++i) {
+ auto& cluster = clustersList->Clusters[i];
+
+ cluster.Name = t.GetList(i).GetStruct(0).GetOptional().GetText();
+ cluster.Datacenter = cluster.Name;
+ cluster.Balancer = t.GetList(i).GetStruct(1).GetOptional().GetText();
+
+ cluster.IsLocal = t.GetList(i).GetStruct(2).GetOptional().GetBool();
+ cluster.IsEnabled = t.GetList(i).GetStruct(3).GetOptional().GetBool();
cluster.Weight = t.GetList(i).GetStruct(4).GetOptional().GetUint64();
- }
-
+ }
+
clustersList->Version = t.GetList(0).GetStruct(5).GetOptional().GetInt64();
-
- ClustersList = std::move(clustersList);
- ClustersListUpdateTimestamp = Ctx().Now();
- }
-
-private:
- TString ListClustersQuery;
- TString PreparedQueryId;
- TClustersList::TConstPtr ClustersList = nullptr;
- TMaybe<TInstant> ClustersListUpdateTimestamp;
+
+ ClustersList = std::move(clustersList);
+ ClustersListUpdateTimestamp = Ctx().Now();
+ }
+
+private:
+ TString ListClustersQuery;
+ TString PreparedQueryId;
+ TClustersList::TConstPtr ClustersList = nullptr;
+ TMaybe<TInstant> ClustersListUpdateTimestamp;
THashSet<TActorId> Subscribers;
TString Database;
-};
-
-NActors::IActor* CreateClusterTracker() {
- return new TClusterTracker();
-}
-
-} // namespace NKikimr::NPQ::NClusterTracker
+};
+
+NActors::IActor* CreateClusterTracker() {
+ return new TClusterTracker();
+}
+
+} // namespace NKikimr::NPQ::NClusterTracker
diff --git a/ydb/core/persqueue/cluster_tracker.h b/ydb/core/persqueue/cluster_tracker.h
index 1c6a5fbed7..2ae06cf915 100644
--- a/ydb/core/persqueue/cluster_tracker.h
+++ b/ydb/core/persqueue/cluster_tracker.h
@@ -1,80 +1,80 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/base/events.h>
-
+
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/actors/core/actorid.h>
#include <library/cpp/actors/core/defs.h>
#include <library/cpp/actors/core/event_local.h>
-
+
#include <library/cpp/monlib/dynamic_counters/counters.h>
-
-#include <util/generic/maybe.h>
-#include <util/generic/ptr.h>
-#include <util/string/builder.h>
-
-#include <vector>
-
-namespace NKikimr::NPQ::NClusterTracker {
-
+
+#include <util/generic/maybe.h>
+#include <util/generic/ptr.h>
+#include <util/string/builder.h>
+
+#include <vector>
+
+namespace NKikimr::NPQ::NClusterTracker {
+
using NActors::TActorId;
-
+
inline TActorId MakeClusterTrackerID() {
- static const char x[12] = "clstr_trckr";
+ static const char x[12] = "clstr_trckr";
return TActorId(0, TStringBuf(x, 12));
-}
-
-NActors::IActor* CreateClusterTracker();
-
-struct TClustersList : public TAtomicRefCount<TClustersList>, TNonCopyable {
- using TConstPtr = TIntrusiveConstPtr<TClustersList>;
-
- TClustersList() = default;
-
- struct TCluster {
- TString Name;
- TString Datacenter; // is equal to cluster name at the moment
- TString Balancer;
- bool IsEnabled = false;
- bool IsLocal = false;
- ui64 Weight = 1000;
-
- TString DebugString() const {
- TStringBuilder builder;
- builder << "(" << Name << ", " << Datacenter << ", " << Balancer << ", ";
- builder << (IsEnabled ? "enabled" : "disabled") << ", ";
- builder << (IsLocal ? "local" : "remote") << ", ";
- builder << Weight << ")";
-
- return TString(builder);
- }
-
- bool operator==(const TCluster& other) const;
- };
-
- bool operator==(const TClustersList& other) const;
-
- std::vector<TCluster> Clusters;
-
- i64 Version = 0;
-};
-
-struct TEvClusterTracker {
- enum EEv {
- EvClustersUpdate = EventSpaceBegin(TKikimrEvents::ES_PQ_CLUSTER_TRACKER),
- EvSubscribe,
- EvEnd
- };
-
- struct TEvClustersUpdate : public NActors::TEventLocal<TEvClustersUpdate, EvClustersUpdate> {
- TClustersList::TConstPtr ClustersList;
- TMaybe<TInstant> ClustersListUpdateTimestamp;
- };
-
- struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> {
- };
-
- static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_CLUSTER_TRACKER), "Unexpected TEvClusterTracker event range");
-};
-
-} // namespace NKikimr::NPQ::NClusterTracker
+}
+
+NActors::IActor* CreateClusterTracker();
+
+struct TClustersList : public TAtomicRefCount<TClustersList>, TNonCopyable {
+ using TConstPtr = TIntrusiveConstPtr<TClustersList>;
+
+ TClustersList() = default;
+
+ struct TCluster {
+ TString Name;
+ TString Datacenter; // is equal to cluster name at the moment
+ TString Balancer;
+ bool IsEnabled = false;
+ bool IsLocal = false;
+ ui64 Weight = 1000;
+
+ TString DebugString() const {
+ TStringBuilder builder;
+ builder << "(" << Name << ", " << Datacenter << ", " << Balancer << ", ";
+ builder << (IsEnabled ? "enabled" : "disabled") << ", ";
+ builder << (IsLocal ? "local" : "remote") << ", ";
+ builder << Weight << ")";
+
+ return TString(builder);
+ }
+
+ bool operator==(const TCluster& other) const;
+ };
+
+ bool operator==(const TClustersList& other) const;
+
+ std::vector<TCluster> Clusters;
+
+ i64 Version = 0;
+};
+
+struct TEvClusterTracker {
+ enum EEv {
+ EvClustersUpdate = EventSpaceBegin(TKikimrEvents::ES_PQ_CLUSTER_TRACKER),
+ EvSubscribe,
+ EvEnd
+ };
+
+ struct TEvClustersUpdate : public NActors::TEventLocal<TEvClustersUpdate, EvClustersUpdate> {
+ TClustersList::TConstPtr ClustersList;
+ TMaybe<TInstant> ClustersListUpdateTimestamp;
+ };
+
+ struct TEvSubscribe : public NActors::TEventLocal<TEvSubscribe, EvSubscribe> {
+ };
+
+ static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_CLUSTER_TRACKER), "Unexpected TEvClusterTracker event range");
+};
+
+} // namespace NKikimr::NPQ::NClusterTracker
diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make
index e2e26d5902..50e89676ef 100644
--- a/ydb/core/persqueue/ya.make
+++ b/ydb/core/persqueue/ya.make
@@ -7,11 +7,11 @@ OWNER(
)
SRCS(
- cluster_tracker.cpp
- blob.cpp
+ cluster_tracker.cpp
+ blob.cpp
event_helpers.cpp
- header.cpp
- percentile_counter.cpp
+ header.cpp
+ percentile_counter.cpp
pq.cpp
pq_database.cpp
pq_impl.cpp
@@ -23,7 +23,7 @@ SRCS(
pq_l2_cache.cpp
read_balancer.cpp
read_speed_limiter.cpp
- subscriber.cpp
+ subscriber.cpp
type_codecs_defs.cpp
user_info.cpp
write_meta.cpp
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index cc725da0cd..d64169d4fc 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -703,25 +703,25 @@ message TSqsConfig {
// Max time value for long polling (param WaitTimeSeconds for ReceiveMessage)
optional uint64 MaxWaitTimeoutMs = 10 [default = 20000];
-
- optional uint32 SchemeCacheSoftRefreshTimeSeconds = 11 [default = 5];
- optional uint32 SchemeCacheHardRefreshTimeSeconds = 12 [default = 10];
-
- optional bool ForceAccessControl = 13 [default = false];
- repeated string AccountsWithoutMandatoryAuth = 14;
-
- optional bool YandexCloudMode = 15 [default = false];
- optional uint32 YandexCloudServiceId = 16; // only 15 lesser bits should be used!
- optional string YandexCloudAccessServiceAddress = 29; // host:port
- optional string YandexCloudFolderServiceAddress = 30; // host:port
- optional string YandexCloudServiceRegion = 39 [default = "ru-central1"];
-
- optional uint32 MeteringFlushingIntervalMs = 51 [default = 5000];
- optional string MeteringLogFilePath = 52;
-
- repeated string MeteringCloudNetCidr = 53;
- repeated string MeteringYandexNetCidr = 54;
-
+
+ optional uint32 SchemeCacheSoftRefreshTimeSeconds = 11 [default = 5];
+ optional uint32 SchemeCacheHardRefreshTimeSeconds = 12 [default = 10];
+
+ optional bool ForceAccessControl = 13 [default = false];
+ repeated string AccountsWithoutMandatoryAuth = 14;
+
+ optional bool YandexCloudMode = 15 [default = false];
+ optional uint32 YandexCloudServiceId = 16; // only 15 lesser bits should be used!
+ optional string YandexCloudAccessServiceAddress = 29; // host:port
+ optional string YandexCloudFolderServiceAddress = 30; // host:port
+ optional string YandexCloudServiceRegion = 39 [default = "ru-central1"];
+
+ optional uint32 MeteringFlushingIntervalMs = 51 [default = 5000];
+ optional string MeteringLogFilePath = 52;
+
+ repeated string MeteringCloudNetCidr = 53;
+ repeated string MeteringYandexNetCidr = 54;
+
// TODO: remove both
optional uint32 MastersDescriberUpdateTimeMs = 17 [default = 10000];
optional uint32 MasterConnectTimeoutMs = 18 [default = 10000];
@@ -741,10 +741,10 @@ message TSqsConfig {
optional uint64 MaxNumberOfReceiveMessages = 24 [default = 10]; // MaxNumberOfMessages parameter in ReceiveMessage
optional uint64 RequestTimeoutMs = 25 [default = 600000];
-
- optional bool ForceQueueCreationV2 = 26 [default = true]; // deprecated, TODO: remove from config
- optional bool ForceQueueDeletionV2 = 27 [default = true]; // deprecated, TODO: remove from config
- optional bool EnableDeadLetterQueues = 41 [default = false];
+
+ optional bool ForceQueueCreationV2 = 26 [default = true]; // deprecated, TODO: remove from config
+ optional bool ForceQueueDeletionV2 = 27 [default = true]; // deprecated, TODO: remove from config
+ optional bool EnableDeadLetterQueues = 41 [default = false];
optional bool CreateLegacyDurationCounters = 28;
@@ -814,7 +814,7 @@ message TSqsConfig {
}
optional TQuotingConfig QuotingConfig = 47;
- optional bool EnableQueueAttributesValidation = 48 [default = true];
+ optional bool EnableQueueAttributesValidation = 48 [default = true];
message TAccountSettingsDefaults {
optional int64 MaxQueuesCount = 1 [default = 50];
@@ -832,13 +832,13 @@ message TSqsConfig {
optional uint64 QueuesListReadBatchSize = 59 [default = 1000];
optional bool ValidateMessageBody = 60;
-
- optional uint64 DlqNotificationGracePeriodMs = 61 [default = 60000];
+
+ optional uint64 DlqNotificationGracePeriodMs = 61 [default = 60000];
optional uint64 AddMessagesToInflyCheckPeriodMs = 62 [default = 30000];
optional uint64 AddMessagesToInflyMinCheckAttempts = 63 [default = 10];
-
- optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600];
+
+ optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600];
optional bool MeteringByNetClassifierOnly = 65 [default = false];
@@ -1370,9 +1370,9 @@ message TAppConfig {
optional NKikimrSharedCache.TSharedCacheConfig SharedCacheConfig = 38; // dynamic configuration via cms
optional TImmediateControlsConfig ImmediateControlsConfig = 39;
optional TAllocatorConfig AllocatorConfig = 40;
- optional NKikimrPQ.TPQClusterDiscoveryConfig PQClusterDiscoveryConfig = 41;
- optional NKikimrNetClassifier.TNetClassifierConfig NetClassifierConfig = 42;
- optional NKikimrNetClassifier.TNetClassifierDistributableConfig NetClassifierDistributableConfig = 43; // also dynamic via cms
+ optional NKikimrPQ.TPQClusterDiscoveryConfig PQClusterDiscoveryConfig = 41;
+ optional NKikimrNetClassifier.TNetClassifierConfig NetClassifierConfig = 42;
+ optional NKikimrNetClassifier.TNetClassifierDistributableConfig NetClassifierDistributableConfig = 43; // also dynamic via cms
optional NKikimrResourceBroker.TResourceBrokerConfig ResourceBrokerConfig = 44;
optional TMeteringConfig MeteringConfig = 45;
optional THiveConfig HiveConfig = 46;
diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto
index ead88eafce..d5a32d05c6 100644
--- a/ydb/core/protos/console_config.proto
+++ b/ydb/core/protos/console_config.proto
@@ -101,10 +101,10 @@ message TConfigItem {
TableServiceConfigItem = 37;
SharedCacheConfigItem = 38;
ImmediateControlsConfigItem = 39;
- AllocatorConfigItem = 40;
- PQClusterDiscoveryConfigItem = 41;
- NetClassifierConfigItem = 42;
- NetClassifierDistributableConfigItem = 43;
+ AllocatorConfigItem = 40;
+ PQClusterDiscoveryConfigItem = 41;
+ NetClassifierConfigItem = 42;
+ NetClassifierDistributableConfigItem = 43;
ResourceBrokerConfigItem = 44;
MeteringConfigItem = 45;
HiveConfigItem = 46;
diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto
index d0f3cfa4d8..df7cda5980 100644
--- a/ydb/core/protos/msgbus.proto
+++ b/ydb/core/protos/msgbus.proto
@@ -608,13 +608,13 @@ message TSqsRequest {
NKikimr.NSQS.TCreateUserRequest CreateUser = 15;
NKikimr.NSQS.TDeleteUserRequest DeleteUser = 16;
NKikimr.NSQS.TListUsersRequest ListUsers = 17;
- NKikimr.NSQS.TModifyPermissionsRequest ModifyPermissions = 18;
- NKikimr.NSQS.TListPermissionsRequest ListPermissions = 19;
+ NKikimr.NSQS.TModifyPermissionsRequest ModifyPermissions = 18;
+ NKikimr.NSQS.TListPermissionsRequest ListPermissions = 19;
NKikimr.NSQS.TDeleteQueueBatchRequest DeleteQueueBatch = 20;
NKikimr.NSQS.TPurgeQueueBatchRequest PurgeQueueBatch = 21;
NKikimr.NSQS.TGetQueueAttributesBatchRequest GetQueueAttributesBatch = 22;
- NKikimr.NSQS.TListDeadLetterSourceQueuesRequest ListDeadLetterSourceQueues = 23;
- NKikimr.NSQS.TCountQueuesRequest CountQueues = 24;
+ NKikimr.NSQS.TListDeadLetterSourceQueuesRequest ListDeadLetterSourceQueues = 23;
+ NKikimr.NSQS.TCountQueuesRequest CountQueues = 24;
}
optional string RequestId = 30;
optional bool RequestRateLimit = 31 [default = true];
@@ -639,18 +639,18 @@ message TSqsResponse {
NKikimr.NSQS.TCreateUserResponse CreateUser = 15;
NKikimr.NSQS.TDeleteUserResponse DeleteUser = 16;
NKikimr.NSQS.TListUsersResponse ListUsers = 17;
- NKikimr.NSQS.TModifyPermissionsResponse ModifyPermissions = 18;
- NKikimr.NSQS.TListPermissionsResponse ListPermissions = 19;
+ NKikimr.NSQS.TModifyPermissionsResponse ModifyPermissions = 18;
+ NKikimr.NSQS.TListPermissionsResponse ListPermissions = 19;
NKikimr.NSQS.TDeleteQueueBatchResponse DeleteQueueBatch = 20;
NKikimr.NSQS.TPurgeQueueBatchResponse PurgeQueueBatch = 21;
NKikimr.NSQS.TGetQueueAttributesBatchResponse GetQueueAttributesBatch = 22;
- NKikimr.NSQS.TListDeadLetterSourceQueuesResponse ListDeadLetterSourceQueues = 23;
- NKikimr.NSQS.TCountQueuesResponse CountQueues = 24;
+ NKikimr.NSQS.TListDeadLetterSourceQueuesResponse ListDeadLetterSourceQueues = 23;
+ NKikimr.NSQS.TCountQueuesResponse CountQueues = 24;
}
optional string RequestId = 30;
- optional string FolderId = 31;
- optional string ResourceId = 32;
- optional bool IsFifo = 33;
+ optional string FolderId = 31;
+ optional string ResourceId = 32;
+ optional bool IsFifo = 33;
}
message TS3ListingRequest {
diff --git a/ydb/core/protos/netclassifier.proto b/ydb/core/protos/netclassifier.proto
index 5cf1f4be15..0c71eae7bf 100644
--- a/ydb/core/protos/netclassifier.proto
+++ b/ydb/core/protos/netclassifier.proto
@@ -1,45 +1,45 @@
-package NKikimrNetClassifier;
-
-option java_package = "ru.yandex.kikimr.proto";
-
-message TNetClassifierUpdaterConfig {
+package NKikimrNetClassifier;
+
+option java_package = "ru.yandex.kikimr.proto";
+
+message TNetClassifierUpdaterConfig {
enum EFormat {
TSV = 0;
NETBOX = 1;
}
- optional string NetDataSourceUrl = 1;
- optional uint32 RetryIntervalSeconds = 2;
- optional uint32 NetDataUpdateIntervalSeconds = 3;
+ optional string NetDataSourceUrl = 1;
+ optional uint32 RetryIntervalSeconds = 2;
+ optional uint32 NetDataUpdateIntervalSeconds = 3;
optional EFormat Format = 4 [default = TSV];
repeated string NetBoxTags = 5;
-}
-
-message TSubnet {
- optional string Mask = 1;
- optional string Label = 2;
-}
-
-message TNetData {
- repeated TSubnet Subnets = 1;
-}
-
-message TNetClassifierConfig {
- optional TNetClassifierUpdaterConfig UpdaterConfig = 1;
-
- // The file content is used if distributable config is empty or unavailable. May be empty
- optional string NetDataFilePath = 2;
-
- // CMS has to respond with distributable configuration before the time runs out
- optional uint32 CmsConfigTimeoutSeconds = 3 [default = 30];
-
- // Timed counters update interval (seconds):
- optional uint32 TimedCountersUpdateIntervalSeconds = 4 [default = 15];
-}
-
-message TNetClassifierDistributableConfig {
- // DO NOT SET the config fields manually
- optional string LastUpdateDatetimeUTC = 1;
- optional uint64 LastUpdateTimestamp = 2;
- optional bytes PackedNetData = 3; // Contains serialized and deflated TNetData
-}
+}
+
+message TSubnet {
+ optional string Mask = 1;
+ optional string Label = 2;
+}
+
+message TNetData {
+ repeated TSubnet Subnets = 1;
+}
+
+message TNetClassifierConfig {
+ optional TNetClassifierUpdaterConfig UpdaterConfig = 1;
+
+ // The file content is used if distributable config is empty or unavailable. May be empty
+ optional string NetDataFilePath = 2;
+
+ // CMS has to respond with distributable configuration before the time runs out
+ optional uint32 CmsConfigTimeoutSeconds = 3 [default = 30];
+
+ // Timed counters update interval (seconds):
+ optional uint32 TimedCountersUpdateIntervalSeconds = 4 [default = 15];
+}
+
+message TNetClassifierDistributableConfig {
+ // DO NOT SET the config fields manually
+ optional string LastUpdateDatetimeUTC = 1;
+ optional uint64 LastUpdateTimestamp = 2;
+ optional bytes PackedNetData = 3; // Contains serialized and deflated TNetData
+}
diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto
index 7b687ed4b3..7c85927449 100644
--- a/ydb/core/protos/pqconfig.proto
+++ b/ydb/core/protos/pqconfig.proto
@@ -1,6 +1,6 @@
import "ydb/public/api/protos/draft/persqueue_error_codes.proto";
import "ydb/public/api/protos/draft/persqueue_common.proto";
-
+
import "ydb/core/protos/base.proto";
import "ydb/core/protos/msgbus_kv.proto";
import "ydb/core/protos/node_limits.proto";
@@ -9,11 +9,11 @@ import "ydb/core/protos/services.proto";
import "library/cpp/actors/protos/actors.proto";
-
+
package NKikimrPQ;
option java_package = "ru.yandex.kikimr.proto";
-
+
message TPartitionMeta {
optional uint64 StartOffset = 1;
optional uint64 EndOffset = 2;
@@ -42,11 +42,11 @@ message TPQConfig {
optional NKikimrClient.TKeyValueRequest.ETactic Tactic = 13 [default = MAX_THROUGHPUT];
optional bool RequireCredentialsInNewProtocol = 14 [default = false];
-
- optional string ClusterTablePath = 15 [default = "/Root/PQ/Config/V2/Cluster"];
- optional string VersionTablePath = 16 [default = "/Root/PQ/Config/V2/Versions"];
-
- optional uint32 ClustersUpdateTimeoutOnErrorSec = 17 [default = 1];
+
+ optional string ClusterTablePath = 15 [default = "/Root/PQ/Config/V2/Cluster"];
+ optional string VersionTablePath = 16 [default = "/Root/PQ/Config/V2/Versions"];
+
+ optional uint32 ClustersUpdateTimeoutOnErrorSec = 17 [default = 1];
optional uint32 WriteInitLatencyBigMs = 19 [default = 900];
optional uint32 ReadInitLatencyBigMs = 20 [default = 900];
@@ -675,13 +675,13 @@ message TClientInfoResponse {
optional uint64 ResponseTimestamp = 4;
repeated TClientInfo ClientInfo = 5;
}
-
-message TPQClusterDiscoveryConfig {
- optional bool Enabled = 1;
- optional uint32 TimedCountersUpdateIntervalSeconds = 2 [default = 15];
- optional NKikimrNetClassifier.TNetData CloudNetData = 3;
+
+message TPQClusterDiscoveryConfig {
+ optional bool Enabled = 1;
+ optional uint32 TimedCountersUpdateIntervalSeconds = 2 [default = 15];
+ optional NKikimrNetClassifier.TNetData CloudNetData = 3;
optional uint64 RequestInflightLimit = 4;
-}
+}
message TYdsNextToken {
required uint64 CreationTimestamp = 1;
diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto
index 3e24042da3..c17c8a7dc3 100644
--- a/ydb/core/protos/services.proto
+++ b/ydb/core/protos/services.proto
@@ -256,10 +256,10 @@ enum EServiceKikimr {
YF_DESCRIPTIONS_SERVICE = 709;
YF_WORKER = 710;
- NET_CLASSIFIER = 711;
-
- PERSQUEUE_CLUSTER_TRACKER = 712;
-
+ NET_CLASSIFIER = 711;
+
+ PERSQUEUE_CLUSTER_TRACKER = 712;
+
YQL_PROXY = 713;
YQL_PRIVATE_PROXY = 714;
@@ -678,13 +678,13 @@ message TActivity {
BS_MONSTREAM_ACTOR = 363;
NODE_WARDEN_STATAGGR_ACTOR = 364;
UPLOAD_ROWS_INTERNAL = 365;
- PERSQUEUE_CLUSTER_TRACKER = 366;
- NET_CLASSIFIER_ACTOR = 367;
+ PERSQUEUE_CLUSTER_TRACKER = 366;
+ NET_CLASSIFIER_ACTOR = 367;
KQP_COMPILE_SERVICE = 368;
KQP_COMPILE_ACTOR = 369;
- PERSQUEUE_CLUSTER_DISCOVERY = 370;
+ PERSQUEUE_CLUSTER_DISCOVERY = 370;
TX_FILL_INDEX_SCAN = 371;
- NET_CLASSIFIER_UPDATER = 372;
+ NET_CLASSIFIER_UPDATER = 372;
BS_LOAD_PDISK_LOG_WRITE = 373;
RTMR_CONFIGURATION_STORAGE = 374;
KQP_SYSTEM_VIEW_SCAN = 375;
diff --git a/ydb/core/protos/sqs.proto b/ydb/core/protos/sqs.proto
index c8a83ee484..eae557a624 100644
--- a/ydb/core/protos/sqs.proto
+++ b/ydb/core/protos/sqs.proto
@@ -4,14 +4,14 @@ message TAuthentification {
optional string UserName = 1;
}
-message TCredentials {
- oneof AccessKey {
- string OAuthToken = 1;
- string TvmTicket = 2;
- string StaticCreds = 3; // Cloud only
- }
-}
-
+message TCredentials {
+ oneof AccessKey {
+ string OAuthToken = 1;
+ string TvmTicket = 2;
+ string StaticCreds = 3; // Cloud only
+ }
+}
+
message TError {
message TDebug {
optional uint32 TransactionStatusCode = 1;
@@ -38,7 +38,7 @@ message TMessageAttribute {
}
message TChangeMessageVisibilityRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// Name of queue.
@@ -60,7 +60,7 @@ message TChangeMessageVisibilityResponse {
}
message TChangeMessageVisibilityBatchRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// Name of queue.
@@ -75,7 +75,7 @@ message TChangeMessageVisibilityBatchResponse {
}
message TCreateQueueRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// Name of queue. Should be unique for specified user.
@@ -90,7 +90,7 @@ message TCreateQueueRequest {
optional bool EnableAutosplit = 8;
/// Size in bytes when partition will be splitted on two parts.
optional uint64 SizeToSplit = 9 [default = 1073741824]; // 1GB
- /// Enable internal tables' transactions out of order execution.
+ /// Enable internal tables' transactions out of order execution.
optional bool EnableOutOfOrderTransactionsExecution = 6 [default = true];
}
@@ -102,7 +102,7 @@ message TCreateQueueResponse {
}
message TCreateUserRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// Then user name we want to create.
@@ -115,7 +115,7 @@ message TCreateUserResponse {
}
message TGetQueueAttributesRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
optional string QueueName = 2;
@@ -157,10 +157,10 @@ message TGetQueueAttributesResponse {
optional bool FifoQueue = 13;
/// Is the content-based deduplication is enabled.
optional bool ContentBasedDeduplication = 14;
- /// Contains maxReceiveCount and target DLQ ARN, defines DLQ params
- optional string RedrivePolicy = 16;
- /// Contains queue ARN
- optional string QueueArn = 17;
+ /// Contains maxReceiveCount and target DLQ ARN, defines DLQ params
+ optional string RedrivePolicy = 16;
+ /// Contains queue ARN
+ optional string QueueArn = 17;
}
message TGetQueueAttributesBatchRequest {
@@ -179,7 +179,7 @@ message TGetQueueAttributesBatchResponse {
}
message TGetQueueUrlRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// The name of the queue whose URL must be fetched.
@@ -193,7 +193,7 @@ message TGetQueueUrlResponse {
}
message TDeleteMessageRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
optional string QueueName = 2;
@@ -210,7 +210,7 @@ message TDeleteMessageResponse {
}
message TDeleteMessageBatchRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
optional string QueueName = 2;
@@ -224,7 +224,7 @@ message TDeleteMessageBatchResponse {
}
message TDeleteQueueRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
optional string QueueName = 2;
@@ -253,7 +253,7 @@ message TDeleteQueueBatchResponse {
}
message TDeleteUserRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// Then user name we want to delete.
@@ -266,7 +266,7 @@ message TDeleteUserResponse {
}
message TListQueuesRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// A string to use for filtering the list results.
@@ -286,7 +286,7 @@ message TListQueuesResponse {
}
message TListUsersRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// A string to use for filtering the list results.
@@ -302,7 +302,7 @@ message TListUsersResponse {
}
message TPurgeQueueRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
optional string QueueName = 2;
@@ -331,7 +331,7 @@ message TPurgeQueueBatchResponse {
}
message TReceiveMessageRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
optional string QueueName = 2;
@@ -384,8 +384,8 @@ message TReceiveMessageResponse {
optional uint64 SentTimestamp = 10;
/// The large, non-consecutive number that SQS assigns to each message.
optional uint64 SequenceNumber = 11;
- /// Message sender's SID, may be empty in case of unauthenticated writes.
- optional string SenderId = 13;
+ /// Message sender's SID, may be empty in case of unauthenticated writes.
+ optional string SenderId = 13;
}
optional TError Error = 1;
@@ -398,7 +398,7 @@ message TReceiveMessageResponse {
}
message TSendMessageRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// The name of the queue in which the message should be placed.
@@ -434,7 +434,7 @@ message TSendMessageResponse {
}
message TSendMessageBatchRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// The name of the queue in which the message should be placed.
@@ -449,7 +449,7 @@ message TSendMessageBatchResponse {
}
message TSetQueueAttributesRequest {
- optional TCredentials Credentials = 100;
+ optional TCredentials Credentials = 100;
/// Sender's authentification.
optional TAuthentification Auth = 1;
/// The name of the queue.
@@ -462,90 +462,90 @@ message TSetQueueAttributesResponse {
optional TError Error = 1;
optional string RequestId = 2;
}
-
-message TPermissions {
- optional string Subject = 1;
- repeated string PermissionNames = 2; // https://st.yandex-team.ru/SQS-3
-}
-
-message TPermissionsAction {
- oneof Action {
- TPermissions Grant = 1;
- TPermissions Revoke = 2;
- TPermissions Set = 3;
- }
-}
-
-message TModifyPermissionsRequest {
- optional TCredentials Credentials = 100;
- optional string Resource = 1;
- repeated TPermissionsAction Actions = 2;
- optional bool ClearACL = 3 [default = false];
- optional TAuthentification Auth = 4; // for compatibility only
-}
-
-message TModifyPermissionsResponse {
- optional TError Error = 1;
- optional string RequestId = 2;
-}
-
-message TListPermissionsRequest {
- optional TCredentials Credentials = 100;
- optional string Path = 1;
- optional TAuthentification Auth = 2; // for compatibility only
-}
-
-message TQueuePermissions {
- repeated TPermissions Permissions = 1;
- repeated TPermissions EffectivePermissions = 2;
-}
-
-message TAccountPermissions {
- repeated TPermissions Permissions = 1;
- repeated TPermissions EffectivePermissions = 2;
-}
-
-message TDirectoryPermissions {
- repeated TPermissions Permissions = 1;
- repeated TPermissions EffectivePermissions = 2;
-}
-
-message TListPermissionsResponse {
- optional TError Error = 1;
- optional string RequestId = 2;
-
- oneof NodeType {
- TQueuePermissions QueuePermissions = 3;
- TQueuePermissions AccountPermissions = 4;
- TQueuePermissions DirectoryPermissions = 5; // not used right now
- }
-}
-
-message TListDeadLetterSourceQueuesRequest {
- optional TCredentials Credentials = 100;
- /// The name of the queue.
- optional string QueueName = 1;
+
+message TPermissions {
+ optional string Subject = 1;
+ repeated string PermissionNames = 2; // https://st.yandex-team.ru/SQS-3
+}
+
+message TPermissionsAction {
+ oneof Action {
+ TPermissions Grant = 1;
+ TPermissions Revoke = 2;
+ TPermissions Set = 3;
+ }
+}
+
+message TModifyPermissionsRequest {
+ optional TCredentials Credentials = 100;
+ optional string Resource = 1;
+ repeated TPermissionsAction Actions = 2;
+ optional bool ClearACL = 3 [default = false];
+ optional TAuthentification Auth = 4; // for compatibility only
+}
+
+message TModifyPermissionsResponse {
+ optional TError Error = 1;
+ optional string RequestId = 2;
+}
+
+message TListPermissionsRequest {
+ optional TCredentials Credentials = 100;
+ optional string Path = 1;
+ optional TAuthentification Auth = 2; // for compatibility only
+}
+
+message TQueuePermissions {
+ repeated TPermissions Permissions = 1;
+ repeated TPermissions EffectivePermissions = 2;
+}
+
+message TAccountPermissions {
+ repeated TPermissions Permissions = 1;
+ repeated TPermissions EffectivePermissions = 2;
+}
+
+message TDirectoryPermissions {
+ repeated TPermissions Permissions = 1;
+ repeated TPermissions EffectivePermissions = 2;
+}
+
+message TListPermissionsResponse {
+ optional TError Error = 1;
+ optional string RequestId = 2;
+
+ oneof NodeType {
+ TQueuePermissions QueuePermissions = 3;
+ TQueuePermissions AccountPermissions = 4;
+ TQueuePermissions DirectoryPermissions = 5; // not used right now
+ }
+}
+
+message TListDeadLetterSourceQueuesRequest {
+ optional TCredentials Credentials = 100;
+ /// The name of the queue.
+ optional string QueueName = 1;
optional TAuthentification Auth = 2;
-}
-
-message TListDeadLetterSourceQueuesResponse {
- message TQueueType {
- optional string QueueName = 1;
- optional string QueueUrl = 2;
- }
- optional TError Error = 1;
- optional string RequestId = 2;
- repeated TQueueType Queues = 3;
-}
-
-message TCountQueuesRequest {
- optional TCredentials Credentials = 100;
- optional TAuthentification Auth = 1;
-}
-
-message TCountQueuesResponse {
- optional TError Error = 1;
- optional string RequestId = 2;
-
- optional uint64 Count = 3;
-}
+}
+
+message TListDeadLetterSourceQueuesResponse {
+ message TQueueType {
+ optional string QueueName = 1;
+ optional string QueueUrl = 2;
+ }
+ optional TError Error = 1;
+ optional string RequestId = 2;
+ repeated TQueueType Queues = 3;
+}
+
+message TCountQueuesRequest {
+ optional TCredentials Credentials = 100;
+ optional TAuthentification Auth = 1;
+}
+
+message TCountQueuesResponse {
+ optional TError Error = 1;
+ optional string RequestId = 2;
+
+ optional uint64 Count = 3;
+}
diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make
index 693ae8ef87..70bb65514c 100644
--- a/ydb/core/protos/ya.make
+++ b/ydb/core/protos/ya.make
@@ -78,7 +78,7 @@ SRCS(
msgbus_health.proto
msgbus_kv.proto
msgbus_pq.proto
- netclassifier.proto
+ netclassifier.proto
node_broker.proto
node_limits.proto
profiler.proto
diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp
index 4804dec8d8..63f0436bff 100644
--- a/ydb/core/testlib/actors/test_runtime.cpp
+++ b/ydb/core/testlib/actors/test_runtime.cpp
@@ -126,7 +126,7 @@ namespace NActors {
nodeAppData->PollerThreads = node->Poller;
nodeAppData->StreamingConfig.SetEnableOutputStreams(true);
nodeAppData->PQConfig = app0->PQConfig;
- nodeAppData->NetClassifierConfig.CopyFrom(app0->NetClassifierConfig);
+ nodeAppData->NetClassifierConfig.CopyFrom(app0->NetClassifierConfig);
nodeAppData->StaticBlobStorageConfig->CopyFrom(*app0->StaticBlobStorageConfig);
nodeAppData->EnableKqpSpilling = app0->EnableKqpSpilling;
nodeAppData->FeatureFlags = app0->FeatureFlags;
diff --git a/ydb/core/testlib/basics/appdata.h b/ydb/core/testlib/basics/appdata.h
index b65385b077..409869161f 100644
--- a/ydb/core/testlib/basics/appdata.h
+++ b/ydb/core/testlib/basics/appdata.h
@@ -75,7 +75,7 @@ namespace NKikimr {
TMap<ui32, NKikimrProto::TKeyConfig> Keys;
bool EnableKqpSpilling = false;
NKikimrConfig::TCompactionConfig CompactionConfig;
- TString NetDataSourceUrl;
+ TString NetDataSourceUrl;
NKikimrConfig::THiveConfig HiveConfig;
NKikimrConfig::TDataShardConfig DataShardConfig;
NKikimrConfig::TMeteringConfig MeteringConfig;
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index c477916952..d4907c26f1 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -158,7 +158,7 @@ namespace Tests {
return;
TAppPrepare app; /* will cook TAppData */
- app.SetNetDataSourceUrl(Settings->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl());
+ app.SetNetDataSourceUrl(Settings->NetClassifierConfig.GetUpdaterConfig().GetNetDataSourceUrl());
app.SetEnableKqpSpilling(Settings->EnableKqpSpilling);
app.SetKeepSnapshotTimeout(Settings->KeepSnapshotTimeout);
app.SetChangesQueueItemsLimit(Settings->ChangesQueueItemsLimit);
@@ -255,8 +255,8 @@ namespace Tests {
auto grpcMon = system->Register(NGRpcService::CreateGrpcMonService(), TMailboxType::ReadAsFilled);
system->RegisterLocalService(NGRpcService::GrpcMonServiceId(), grpcMon);
- GRpcServerRootCounters = MakeIntrusive<NMonitoring::TDynamicCounters>();
- auto& counters = GRpcServerRootCounters;
+ GRpcServerRootCounters = MakeIntrusive<NMonitoring::TDynamicCounters>();
+ auto& counters = GRpcServerRootCounters;
auto& appData = Runtime->GetAppData();
@@ -307,7 +307,7 @@ namespace Tests {
GRpcServer->AddService(new NGRpcService::TGRpcYdbScriptingService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::TGRpcOperationService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::V1::TGRpcPersQueueService(system, counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId));
- GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId));
+ GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NKesus::TKesusGRpcService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::TGRpcCmsService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::TGRpcDiscoveryService(system, counters, grpcRequestProxyId));
@@ -692,13 +692,13 @@ namespace Tests {
TActorId kesusServiceId = Runtime->Register(kesusService, nodeIdx);
Runtime->RegisterService(NKesus::MakeKesusProxyServiceId(), kesusServiceId, nodeIdx);
}
-
+
{
IActor* pqClusterTracker = NPQ::NClusterTracker::CreateClusterTracker();
TActorId pqClusterTrackerId = Runtime->Register(pqClusterTracker, nodeIdx);
Runtime->RegisterService(NPQ::NClusterTracker::MakeClusterTrackerID(), pqClusterTrackerId, nodeIdx);
}
-
+
{
IActor* netClassifier = NNetClassifier::CreateNetClassifier();
TActorId netClassifierId = Runtime->Register(netClassifier, nodeIdx);
diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h
index 18af6f49c9..2064752ab4 100644
--- a/ydb/core/testlib/test_client.h
+++ b/ydb/core/testlib/test_client.h
@@ -91,8 +91,8 @@ namespace Tests {
ui16 GrpcPort = 0;
NKikimrProto::TAuthConfig AuthConfig;
NKikimrPQ::TPQConfig PQConfig;
- NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig;
- NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig;
+ NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig;
+ NKikimrNetClassifier::TNetClassifierConfig NetClassifierConfig;
ui32 Domain = TestDomain;
bool SupportsRedirect = true;
TString TracePath;
@@ -222,10 +222,10 @@ namespace Tests {
void SetupDefaultProfiles();
- TIntrusivePtr<NMonitoring::TDynamicCounters> GetGRpcServerRootCounters() const {
- return GRpcServerRootCounters;
- }
-
+ TIntrusivePtr<NMonitoring::TDynamicCounters> GetGRpcServerRootCounters() const {
+ return GRpcServerRootCounters;
+ }
+
void ShutdownGRpc() {
if (GRpcServer) {
GRpcServer->Stop();
@@ -257,7 +257,7 @@ namespace Tests {
const NBus::TBusServerSessionConfig BusServerSessionConfig; //BusServer hold const & on config
TAutoPtr<NMsgBusProxy::IMessageBusServer> BusServer;
std::unique_ptr<NGrpc::TGRpcServer> GRpcServer;
- TIntrusivePtr<NMonitoring::TDynamicCounters> GRpcServerRootCounters;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> GRpcServerRootCounters;
};
class TClient {
diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h
index 7b00ae5c5e..f059923250 100644
--- a/ydb/core/testlib/test_pq_client.h
+++ b/ydb/core/testlib/test_pq_client.h
@@ -14,9 +14,9 @@
#include <library/cpp/tvmauth/unittest.h>
#include <library/cpp/testing/unittest/registar.h>
-#include <util/string/printf.h>
+#include <util/string/printf.h>
#include <util/system/tempfile.h>
-
+
namespace NKikimr {
namespace NPersQueueTests {
@@ -612,7 +612,7 @@ public:
RunYqlSchemeQuery(R"___(
CREATE TABLE [/Root/PQ/Config/V2/Cluster] (
name Utf8,
- balancer Utf8,
+ balancer Utf8,
local Bool,
enabled Bool,
weight Uint64,
@@ -626,11 +626,11 @@ public:
)___");
RunYqlSchemeQuery(R"___(
- CREATE TABLE [/Root/PQ/Config/V2/Versions] (
- name Utf8,
- version Int64,
- PRIMARY KEY (name)
- );
+ CREATE TABLE [/Root/PQ/Config/V2/Versions] (
+ name Utf8,
+ version Int64,
+ PRIMARY KEY (name)
+ );
)___");
TStringBuilder upsertClusters;
@@ -700,25 +700,25 @@ public:
("user5", "1"),
("user3", "2");
UPSERT INTO [/Root/PQ/Config/V2/Producer] (name, tvmClientId) VALUES
- ("user4", "2"),
- ("topic1", "1");
+ ("user4", "2"),
+ ("topic1", "1");
)___");
}
- void UpdateDC(const TString& name, bool local, bool enabled) {
- const TString query = Sprintf(
- R"___(
- UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, local, enabled) VALUES
- ("%s", %s, %s);
- UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version)
- SELECT name, version + 1 FROM [/Root/PQ/Config/V2/Versions] WHERE name == "Cluster";
- )___", name.c_str(), (local ? "true" : "false"), (enabled ? "true" : "false"));
-
+ void UpdateDC(const TString& name, bool local, bool enabled) {
+ const TString query = Sprintf(
+ R"___(
+ UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, local, enabled) VALUES
+ ("%s", %s, %s);
+ UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version)
+ SELECT name, version + 1 FROM [/Root/PQ/Config/V2/Versions] WHERE name == "Cluster";
+ )___", name.c_str(), (local ? "true" : "false"), (enabled ? "true" : "false"));
+
RunYqlDataQuery(query);
- }
-
+ }
+
void DisableDC() {
- UpdateDC("dc1", true, false);
+ UpdateDC("dc1", true, false);
}
void RestartSchemeshard(TTestActorRuntime* runtime) {
diff --git a/ydb/core/util/address_classifier.cpp b/ydb/core/util/address_classifier.cpp
index 01737dc931..46f291e417 100644
--- a/ydb/core/util/address_classifier.cpp
+++ b/ydb/core/util/address_classifier.cpp
@@ -1,96 +1,96 @@
-#include "address_classifier.h"
-
-namespace NKikimr::NAddressClassifier {
-
-TString ExtractAddress(const TString& peer) {
- TStringBuf buf(peer);
+#include "address_classifier.h"
+
+namespace NKikimr::NAddressClassifier {
+
+TString ExtractAddress(const TString& peer) {
+ TStringBuf buf(peer);
if (buf.SkipPrefix(TStringBuf("ipv"))) {
- buf.Skip(2); // skip 4/6 and ':'
- if (buf.StartsWith('[')) {
- buf.Skip(1);
- buf = buf.Before(']');
- return TString(buf);
- }
- }
-
- buf = buf.RBefore(':'); // remove port
-
- return TString(buf);
-}
-
-bool TAddressClassifier::AddNetByCidrAndLabel(const TString& cidr, const size_t label) {
- try {
- // The following method still throws despite its name
- const auto maybeRange = TIpAddressRange::TryFromCidrString(cidr);
- if (maybeRange.Empty()) {
- return false;
- }
-
- if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv4) {
- RegisteredV4Nets[label].Add(*maybeRange);
- } else if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv6) {
- RegisteredV6Nets[label].Add(*maybeRange);
- } else {
- Y_VERIFY(false); // unknown net type?
- }
- } catch (yexception&) {
- return false;
- }
-
- return true;
-}
-
-std::pair<bool, size_t> TAddressClassifier::ClassifyAddress(const TString& address) const {
- bool ok;
- const auto inetAddress = TIpv6Address::FromString(address, ok);
- if (!ok) {
- return UnknownAddressClass;
- }
-
- auto findAddress = [&inetAddress](const auto& registeredNets) -> std::pair<bool, size_t> {
- for (const auto& [label, rangeSet] : registeredNets) {
- if (rangeSet.Contains(inetAddress)) {
- return {true, label};
- }
- }
- return UnknownAddressClass;
- };
-
- if (inetAddress.Type() == TIpv6Address::TIpType::Ipv4) {
- return findAddress(RegisteredV4Nets);
- } else if (inetAddress.Type() == TIpv6Address::TIpType::Ipv6) {
- return findAddress(RegisteredV6Nets);
- } else {
- Y_VERIFY(false); // unknown net type?
- }
-
- return UnknownAddressClass;
-}
-
-TLabeledAddressClassifier::TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels)
- : Classifier(std::move(addressClassifier))
- , Labels(std::move(labels))
- {
- }
-
-TLabeledAddressClassifier::TConstPtr TLabeledAddressClassifier::MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels) {
- return MakeIntrusiveConst<TLabeledAddressClassifier>(std::move(addressClassifier), std::move(labels));
-}
-
-TMaybe<TString> TLabeledAddressClassifier::ClassifyAddress(const TString& address) const {
- TMaybe<TString> result = Nothing();
-
- const auto netClass = Classifier.ClassifyAddress(address);
- if (netClass != TAddressClassifier::UnknownAddressClass) {
- Y_VERIFY(netClass.second < Labels.size());
- result = Labels[netClass.second];
- }
-
- return result;
-}
-
-const std::vector<TString>& TLabeledAddressClassifier::GetLabels() const {
- return Labels;
-}
-
-} // namespace NKikimr::NAddressClassifier
+ buf.Skip(2); // skip 4/6 and ':'
+ if (buf.StartsWith('[')) {
+ buf.Skip(1);
+ buf = buf.Before(']');
+ return TString(buf);
+ }
+ }
+
+ buf = buf.RBefore(':'); // remove port
+
+ return TString(buf);
+}
+
+bool TAddressClassifier::AddNetByCidrAndLabel(const TString& cidr, const size_t label) {
+ try {
+ // The following method still throws despite its name
+ const auto maybeRange = TIpAddressRange::TryFromCidrString(cidr);
+ if (maybeRange.Empty()) {
+ return false;
+ }
+
+ if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv4) {
+ RegisteredV4Nets[label].Add(*maybeRange);
+ } else if (maybeRange->Type() == TIpAddressRange::TIpType::Ipv6) {
+ RegisteredV6Nets[label].Add(*maybeRange);
+ } else {
+ Y_VERIFY(false); // unknown net type?
+ }
+ } catch (yexception&) {
+ return false;
+ }
+
+ return true;
+}
+
+std::pair<bool, size_t> TAddressClassifier::ClassifyAddress(const TString& address) const {
+ bool ok;
+ const auto inetAddress = TIpv6Address::FromString(address, ok);
+ if (!ok) {
+ return UnknownAddressClass;
+ }
+
+ auto findAddress = [&inetAddress](const auto& registeredNets) -> std::pair<bool, size_t> {
+ for (const auto& [label, rangeSet] : registeredNets) {
+ if (rangeSet.Contains(inetAddress)) {
+ return {true, label};
+ }
+ }
+ return UnknownAddressClass;
+ };
+
+ if (inetAddress.Type() == TIpv6Address::TIpType::Ipv4) {
+ return findAddress(RegisteredV4Nets);
+ } else if (inetAddress.Type() == TIpv6Address::TIpType::Ipv6) {
+ return findAddress(RegisteredV6Nets);
+ } else {
+ Y_VERIFY(false); // unknown net type?
+ }
+
+ return UnknownAddressClass;
+}
+
+TLabeledAddressClassifier::TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels)
+ : Classifier(std::move(addressClassifier))
+ , Labels(std::move(labels))
+ {
+ }
+
+TLabeledAddressClassifier::TConstPtr TLabeledAddressClassifier::MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels) {
+ return MakeIntrusiveConst<TLabeledAddressClassifier>(std::move(addressClassifier), std::move(labels));
+}
+
+TMaybe<TString> TLabeledAddressClassifier::ClassifyAddress(const TString& address) const {
+ TMaybe<TString> result = Nothing();
+
+ const auto netClass = Classifier.ClassifyAddress(address);
+ if (netClass != TAddressClassifier::UnknownAddressClass) {
+ Y_VERIFY(netClass.second < Labels.size());
+ result = Labels[netClass.second];
+ }
+
+ return result;
+}
+
+const std::vector<TString>& TLabeledAddressClassifier::GetLabels() const {
+ return Labels;
+}
+
+} // namespace NKikimr::NAddressClassifier
diff --git a/ydb/core/util/address_classifier.h b/ydb/core/util/address_classifier.h
index bf4246dc3b..f5e25cd3e4 100644
--- a/ydb/core/util/address_classifier.h
+++ b/ydb/core/util/address_classifier.h
@@ -1,75 +1,75 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/ipmath/range_set.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-
-#include <vector>
-
-namespace NKikimr::NAddressClassifier {
-
-TString ExtractAddress(const TString& grpcPeer);
-
-class TAddressClassifier {
-public:
- bool AddNetByCidrAndLabel(const TString& cidr, const size_t label);
- std::pair<bool, size_t> ClassifyAddress(const TString& address) const;
-
- static constexpr std::pair<bool, size_t> UnknownAddressClass = {false, 0};
-
- TAddressClassifier() = default;
- TAddressClassifier(TAddressClassifier&&) = default;
- TAddressClassifier(const TAddressClassifier&) = delete;
-
-private:
- THashMap<int, TIpRangeSet> RegisteredV4Nets;
- THashMap<int, TIpRangeSet> RegisteredV6Nets;
-};
-
-class TLabeledAddressClassifier : public TAtomicRefCount<TLabeledAddressClassifier>, TNonCopyable {
-public:
- using TConstPtr = TIntrusiveConstPtr<TLabeledAddressClassifier>;
-
-public:
- TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels);
- static TConstPtr MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels);
- TMaybe<TString> ClassifyAddress(const TString& address) const;
-
- const std::vector<TString>& GetLabels() const;
-
-private:
- TAddressClassifier Classifier;
- std::vector<TString> Labels;
-};
-
-template <typename TNetData>
-TLabeledAddressClassifier::TConstPtr BuildLabeledAddressClassifierFromNetData(const TNetData& netData) {
- TAddressClassifier addressClassifier;
- THashMap<TString, size_t> knownLabels;
- std::vector<TString> labels;
-
- for (size_t i = 0; i < netData.SubnetsSize(); ++i) {
- const auto& subnet = netData.GetSubnets(i);
-
- auto it = knownLabels.find(subnet.GetLabel());
-
- size_t subnetIntLabel = 0;
- if (it == knownLabels.end()) {
- labels.push_back(subnet.GetLabel());
- subnetIntLabel = labels.size() - 1;
- knownLabels.insert(std::make_pair(labels.back(), subnetIntLabel));
- } else {
- subnetIntLabel = it->second;
- }
-
- if (!addressClassifier.AddNetByCidrAndLabel(subnet.GetMask(), subnetIntLabel)) {
- return nullptr;
- }
- }
-
- return TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels));
-}
-
-} // namespace NKikimr::NAddressClassifier
+
+#include <util/generic/hash.h>
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+
+#include <vector>
+
+namespace NKikimr::NAddressClassifier {
+
+TString ExtractAddress(const TString& grpcPeer);
+
+class TAddressClassifier {
+public:
+ bool AddNetByCidrAndLabel(const TString& cidr, const size_t label);
+ std::pair<bool, size_t> ClassifyAddress(const TString& address) const;
+
+ static constexpr std::pair<bool, size_t> UnknownAddressClass = {false, 0};
+
+ TAddressClassifier() = default;
+ TAddressClassifier(TAddressClassifier&&) = default;
+ TAddressClassifier(const TAddressClassifier&) = delete;
+
+private:
+ THashMap<int, TIpRangeSet> RegisteredV4Nets;
+ THashMap<int, TIpRangeSet> RegisteredV6Nets;
+};
+
+class TLabeledAddressClassifier : public TAtomicRefCount<TLabeledAddressClassifier>, TNonCopyable {
+public:
+ using TConstPtr = TIntrusiveConstPtr<TLabeledAddressClassifier>;
+
+public:
+ TLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels);
+ static TConstPtr MakeLabeledAddressClassifier(TAddressClassifier&& addressClassifier, std::vector<TString>&& labels);
+ TMaybe<TString> ClassifyAddress(const TString& address) const;
+
+ const std::vector<TString>& GetLabels() const;
+
+private:
+ TAddressClassifier Classifier;
+ std::vector<TString> Labels;
+};
+
+template <typename TNetData>
+TLabeledAddressClassifier::TConstPtr BuildLabeledAddressClassifierFromNetData(const TNetData& netData) {
+ TAddressClassifier addressClassifier;
+ THashMap<TString, size_t> knownLabels;
+ std::vector<TString> labels;
+
+ for (size_t i = 0; i < netData.SubnetsSize(); ++i) {
+ const auto& subnet = netData.GetSubnets(i);
+
+ auto it = knownLabels.find(subnet.GetLabel());
+
+ size_t subnetIntLabel = 0;
+ if (it == knownLabels.end()) {
+ labels.push_back(subnet.GetLabel());
+ subnetIntLabel = labels.size() - 1;
+ knownLabels.insert(std::make_pair(labels.back(), subnetIntLabel));
+ } else {
+ subnetIntLabel = it->second;
+ }
+
+ if (!addressClassifier.AddNetByCidrAndLabel(subnet.GetMask(), subnetIntLabel)) {
+ return nullptr;
+ }
+ }
+
+ return TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels));
+}
+
+} // namespace NKikimr::NAddressClassifier
diff --git a/ydb/core/util/address_classifier_ut.cpp b/ydb/core/util/address_classifier_ut.cpp
index 0fb0513e7a..aaa2c8e3e5 100644
--- a/ydb/core/util/address_classifier_ut.cpp
+++ b/ydb/core/util/address_classifier_ut.cpp
@@ -1,135 +1,135 @@
#include <ydb/core/util/address_classifier.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/string.h>
-
-#include <vector>
-
-namespace NKikimr::NAddressClassifier {
-
-struct TSubnet {
- TString Mask;
- TString Label;
-
- const TString& GetMask() const { return Mask; }
- const TString& GetLabel() const { return Label; }
-};
-
-struct TNetData {
- const auto& GetSubnets(const size_t i) const {
- return Subnets[i];
- }
-
- size_t SubnetsSize() const {
- return Subnets.size();
- }
-
- void AddSubnet(const TString& mask, const TString& label) {
- Subnets.push_back({mask, label});
- }
-
-private:
- std::vector<TSubnet> Subnets;
-};
-
-Y_UNIT_TEST_SUITE(AddressClassifierTest) {
- Y_UNIT_TEST(TestAddressExtraction) {
- UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv6:[::1]:58100"), "::1");
- UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:[192.168.0.1]:1556"), "192.168.0.1");
- UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1:1550"), "192.168.0.1");
- UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1"), "192.168.0.1");
- UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1:1550"), "192.168.0.1");
- UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1"), "192.168.0.1");
- }
-
- Y_UNIT_TEST(TestClassfierWithAllIpTypes) {
- TAddressClassifier classifier;
-
- UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("5....../", 34));
- UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("omglol", 666));
- UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/xx", 777));
-
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 42));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.196.0/24", 42));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 24));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a0d:d6c0::/29", 8));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 10));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 10));
-
- const THashMap<TString, std::pair<bool, size_t>> tests = {
- {"2a02:6b8:bf00::", {true, 42}},
- {"2a02:6b8:af00::", TAddressClassifier::UnknownAddressClass},
-
- {"5.45.196.0", {true, 42}},
- {"5.45.196.1", {true, 42}},
- {"5.45.196.5", {true, 42}},
- {"5.45.195.0", TAddressClassifier::UnknownAddressClass},
-
- {"5.45.217.0", {true, 24}},
- {"5.45.217.192", {true, 24}},
- {"5.45.215.192", TAddressClassifier::UnknownAddressClass},
- {"5.44.217.192", TAddressClassifier::UnknownAddressClass},
-
- {"2a0d:d6c0::", {true, 8}},
- {"2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", {true, 8}},
- {"2a0d:d7c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", TAddressClassifier::UnknownAddressClass},
-
- {"wtflol", TAddressClassifier::UnknownAddressClass},
-
- {"127.255.255.255", {true, 10}},
- {"::1", {true, 10}},
- };
-
- for (const auto& test : tests) {
- UNIT_ASSERT_EQUAL_C(test.second, classifier.ClassifyAddress(test.first), TString("Failed at ") << test.first);
- }
- }
-
- const std::vector<TString> labels = { "local6", "local4", "some6", "some4" };
-
- Y_UNIT_TEST(TestLabeledClassifier) {
- TAddressClassifier classifier;
-
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 0));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 1));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 2));
- UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 3));
- UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("test", 3));
-
- auto labeledClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(classifier), std::vector<TString>(labels));
- UNIT_ASSERT(!labeledClassifier->ClassifyAddress("2a02:6b8:af00::"));
- UNIT_ASSERT(!labeledClassifier->ClassifyAddress("lolwut"));
-
- UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("127.255.255.255"), "local4");
- UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("::1"), "local6");
- UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("5.45.217.192"), "some4");
- UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("2a02:6b8:bf00::"), "some6");
-
- UNIT_ASSERT_EQUAL(labeledClassifier->GetLabels(), labels);
- }
-
- Y_UNIT_TEST(TestLabeledClassifierFromNetData) {
- TAddressClassifier classifier;
-
- TNetData netData;
-
- netData.AddSubnet("::1/128", labels[0]);
- netData.AddSubnet("127.0.0.0/8", labels[1]);
- netData.AddSubnet("2a02:6b8:bf00::/40", labels[2]);
- netData.AddSubnet("5.45.217.0/24", labels[3]);
- netData.AddSubnet("2a0d:d6c0::/29", labels[2]);
-
- auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData);
- UNIT_ASSERT(labeledAddressClassifier);
- UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a0d:d6c0::"), labels[2]);
- UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a02:6b8:bf00::"), labels[2]);
- UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("::1"), labels[0]);
- UNIT_ASSERT_EQUAL(labeledAddressClassifier->GetLabels(), labels);
-
- netData.AddSubnet("invalid subnet", "some label");
- UNIT_ASSERT(!BuildLabeledAddressClassifierFromNetData(netData));
- }
-}
-
-} // namespace NKikimr::NAddressClassifier
+
+#include <util/generic/string.h>
+
+#include <vector>
+
+namespace NKikimr::NAddressClassifier {
+
+struct TSubnet {
+ TString Mask;
+ TString Label;
+
+ const TString& GetMask() const { return Mask; }
+ const TString& GetLabel() const { return Label; }
+};
+
+struct TNetData {
+ const auto& GetSubnets(const size_t i) const {
+ return Subnets[i];
+ }
+
+ size_t SubnetsSize() const {
+ return Subnets.size();
+ }
+
+ void AddSubnet(const TString& mask, const TString& label) {
+ Subnets.push_back({mask, label});
+ }
+
+private:
+ std::vector<TSubnet> Subnets;
+};
+
+Y_UNIT_TEST_SUITE(AddressClassifierTest) {
+ Y_UNIT_TEST(TestAddressExtraction) {
+ UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv6:[::1]:58100"), "::1");
+ UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:[192.168.0.1]:1556"), "192.168.0.1");
+ UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1:1550"), "192.168.0.1");
+ UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("ipv4:192.168.0.1"), "192.168.0.1");
+ UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1:1550"), "192.168.0.1");
+ UNIT_ASSERT_STRINGS_EQUAL(ExtractAddress("192.168.0.1"), "192.168.0.1");
+ }
+
+ Y_UNIT_TEST(TestClassfierWithAllIpTypes) {
+ TAddressClassifier classifier;
+
+ UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("5....../", 34));
+ UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("omglol", 666));
+ UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/xx", 777));
+
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 42));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.196.0/24", 42));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 24));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a0d:d6c0::/29", 8));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 10));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 10));
+
+ const THashMap<TString, std::pair<bool, size_t>> tests = {
+ {"2a02:6b8:bf00::", {true, 42}},
+ {"2a02:6b8:af00::", TAddressClassifier::UnknownAddressClass},
+
+ {"5.45.196.0", {true, 42}},
+ {"5.45.196.1", {true, 42}},
+ {"5.45.196.5", {true, 42}},
+ {"5.45.195.0", TAddressClassifier::UnknownAddressClass},
+
+ {"5.45.217.0", {true, 24}},
+ {"5.45.217.192", {true, 24}},
+ {"5.45.215.192", TAddressClassifier::UnknownAddressClass},
+ {"5.44.217.192", TAddressClassifier::UnknownAddressClass},
+
+ {"2a0d:d6c0::", {true, 8}},
+ {"2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", {true, 8}},
+ {"2a0d:d7c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb", TAddressClassifier::UnknownAddressClass},
+
+ {"wtflol", TAddressClassifier::UnknownAddressClass},
+
+ {"127.255.255.255", {true, 10}},
+ {"::1", {true, 10}},
+ };
+
+ for (const auto& test : tests) {
+ UNIT_ASSERT_EQUAL_C(test.second, classifier.ClassifyAddress(test.first), TString("Failed at ") << test.first);
+ }
+ }
+
+ const std::vector<TString> labels = { "local6", "local4", "some6", "some4" };
+
+ Y_UNIT_TEST(TestLabeledClassifier) {
+ TAddressClassifier classifier;
+
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("::1/128", 0));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("127.0.0.0/8", 1));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("2a02:6b8:bf00::/40", 2));
+ UNIT_ASSERT(classifier.AddNetByCidrAndLabel("5.45.217.0/24", 3));
+ UNIT_ASSERT(!classifier.AddNetByCidrAndLabel("test", 3));
+
+ auto labeledClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(classifier), std::vector<TString>(labels));
+ UNIT_ASSERT(!labeledClassifier->ClassifyAddress("2a02:6b8:af00::"));
+ UNIT_ASSERT(!labeledClassifier->ClassifyAddress("lolwut"));
+
+ UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("127.255.255.255"), "local4");
+ UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("::1"), "local6");
+ UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("5.45.217.192"), "some4");
+ UNIT_ASSERT_STRINGS_EQUAL(*labeledClassifier->ClassifyAddress("2a02:6b8:bf00::"), "some6");
+
+ UNIT_ASSERT_EQUAL(labeledClassifier->GetLabels(), labels);
+ }
+
+ Y_UNIT_TEST(TestLabeledClassifierFromNetData) {
+ TAddressClassifier classifier;
+
+ TNetData netData;
+
+ netData.AddSubnet("::1/128", labels[0]);
+ netData.AddSubnet("127.0.0.0/8", labels[1]);
+ netData.AddSubnet("2a02:6b8:bf00::/40", labels[2]);
+ netData.AddSubnet("5.45.217.0/24", labels[3]);
+ netData.AddSubnet("2a0d:d6c0::/29", labels[2]);
+
+ auto labeledAddressClassifier = BuildLabeledAddressClassifierFromNetData(netData);
+ UNIT_ASSERT(labeledAddressClassifier);
+ UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a0d:d6c0::"), labels[2]);
+ UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("2a02:6b8:bf00::"), labels[2]);
+ UNIT_ASSERT_STRINGS_EQUAL(*labeledAddressClassifier->ClassifyAddress("::1"), labels[0]);
+ UNIT_ASSERT_EQUAL(labeledAddressClassifier->GetLabels(), labels);
+
+ netData.AddSubnet("invalid subnet", "some label");
+ UNIT_ASSERT(!BuildLabeledAddressClassifierFromNetData(netData));
+ }
+}
+
+} // namespace NKikimr::NAddressClassifier
diff --git a/ydb/core/util/ut/ya.make b/ydb/core/util/ut/ya.make
index 7a558fa9b6..315713cad8 100644
--- a/ydb/core/util/ut/ya.make
+++ b/ydb/core/util/ut/ya.make
@@ -20,7 +20,7 @@ PEERDIR(
)
SRCS(
- address_classifier_ut.cpp
+ address_classifier_ut.cpp
bits_ut.cpp
btree_cow_ut.cpp
btree_ut.cpp
diff --git a/ydb/core/util/ya.make b/ydb/core/util/ya.make
index 1debe33bd9..4f325d8fcb 100644
--- a/ydb/core/util/ya.make
+++ b/ydb/core/util/ya.make
@@ -6,7 +6,7 @@ OWNER(
)
SRCS(
- address_classifier.cpp
+ address_classifier.cpp
cache.cpp
cache.h
cache_cache.h
diff --git a/ydb/core/ymq/actor/action.h b/ydb/core/ymq/actor/action.h
index bb4888b301..1801c98d68 100644
--- a/ydb/core/ymq/actor/action.h
+++ b/ydb/core/ymq/actor/action.h
@@ -9,7 +9,7 @@
#include "log.h"
#include "proxy_actor.h"
#include "serviceid.h"
-#include "schema.h"
+#include "schema.h"
#include <ydb/core/base/path.h>
#include <ydb/core/base/ticket_parser.h>
@@ -25,9 +25,9 @@
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
-#include <util/folder/path.h>
+#include <util/folder/path.h>
#include <util/generic/guid.h>
-#include <util/generic/is_in.h>
+#include <util/generic/is_in.h>
#include <util/string/ascii.h>
#include <util/string/join.h>
@@ -66,27 +66,27 @@ public:
return true;
}
- static constexpr bool CreateMissingAccount() {
- return false;
- }
+ static constexpr bool CreateMissingAccount() {
+ return false;
+ }
static constexpr bool NeedUserSpecified() {
return true;
}
void DoCloudBootstrap() {
- if (!SecurityToken_) {
- // TODO: use access service
+ if (!SecurityToken_) {
+ // TODO: use access service
MakeError(MutableErrorDesc(), NErrors::INVALID_CLIENT_TOKEN_ID, "Failed to parse cloud id.");
SendReplyAndDie();
- return;
- }
-
- TStringBuf tokenBuf(SecurityToken_);
- UserName_ = TString(tokenBuf.NextTok(':'));
- FolderId_ = TString(tokenBuf);
- }
-
+ return;
+ }
+
+ TStringBuf tokenBuf(SecurityToken_);
+ UserName_ = TString(tokenBuf.NextTok(':'));
+ FolderId_ = TString(tokenBuf);
+ }
+
void DoBootstrap() {
ui64 configurationFlags = 0;
if (TDerived::NeedQueueAttributes()) {
@@ -102,67 +102,67 @@ public:
GetQueueName(),
configurationFlags)
);
- }
+ }
void CreateAccountOnTheFly() const {
- // TODO: move to separate actor
+ // TODO: move to separate actor
this->Register(
new TCreateUserSchemaActor(Cfg().GetRoot(), UserName_, this->SelfId(), RequestId_, UserCounters_)
- );
- }
-
+ );
+ }
+
void HandleAccountCreated(TSqsEvents::TEvUserCreated::TPtr& ev) {
auto* detailedCounters = UserCounters_ ? UserCounters_->GetDetailedCounters() : nullptr;
- if (ev->Get()->Success) {
+ if (ev->Get()->Success) {
INC_COUNTER(detailedCounters, CreateAccountOnTheFly_Success);
- } else {
+ } else {
RLOG_SQS_ERROR("Failed to create cloud account on the fly. Account name: " << UserName_);
- MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
+ MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
INC_COUNTER(detailedCounters, CreateAccountOnTheFly_Errors);
SendReplyAndDie();
- return;
- }
-
+ return;
+ }
+
DoBootstrap();
- }
-
+ }
+
void Bootstrap(const NActors::TActorContext&) {
RLOG_SQS_DEBUG("Request started. Actor: " << this->SelfId()); // log new request id
StartTs_ = TActivationContext::Now();
-
+
const auto& cfg = Cfg();
-
+
this->Become(&TActionActor::InitialState);
-
+
// Set timeout
if (cfg.GetRequestTimeoutMs()) {
this->Schedule(TDuration::MilliSeconds(cfg.GetRequestTimeoutMs()), new TEvWakeup(REQUEST_TIMEOUT_WAKEUP_TAG), TimeoutCookie_.Get());
}
- if (IsCloud()) {
+ if (IsCloud()) {
DoCloudBootstrap();
-
- if (TDerived::CreateMissingAccount()) {
+
+ if (TDerived::CreateMissingAccount()) {
CreateAccountOnTheFly();
-
- return;
- }
- }
-
+
+ return;
+ }
+ }
+
DoBootstrap();
}
protected:
- template<typename TReq>
- void CopySecurityToken(const TReq& request) {
- SecurityToken_ = ExtractSecurityToken<TReq, TCredentials>(request);
- }
-
- template<typename TReq>
- void CopyAccountName(const TReq& request) {
- UserName_ = request.GetAuth().GetUserName();
- }
-
+ template<typename TReq>
+ void CopySecurityToken(const TReq& request) {
+ SecurityToken_ = ExtractSecurityToken<TReq, TCredentials>(request);
+ }
+
+ template<typename TReq>
+ void CopyAccountName(const TReq& request) {
+ UserName_ = request.GetAuth().GetUserName();
+ }
+
virtual void DoAction() = 0;
virtual TError* MutableErrorDesc() = 0;
@@ -180,11 +180,11 @@ protected:
return true;
}
- virtual bool IsFifoQueue() const {
- Y_VERIFY(IsFifo_);
- return *IsFifo_;
- }
-
+ virtual bool IsFifoQueue() const {
+ Y_VERIFY(IsFifo_);
+ return *IsFifo_;
+ }
+
virtual void DoStart() { }
virtual void DoFinish() { }
@@ -289,13 +289,13 @@ protected:
PrintSlowRequestWarning();
}
Finish();
-
+
if (Cfg().GetYandexCloudMode()) {
- Response_.SetFolderId(FolderId_);
- Response_.SetIsFifo(IsFifo_ ? *IsFifo_ : false);
- Response_.SetResourceId(GetQueueName());
- }
-
+ Response_.SetFolderId(FolderId_);
+ Response_.SetIsFifo(IsFifo_ ? *IsFifo_ : false);
+ Response_.SetResourceId(GetQueueName());
+ }
+
Cb_->DoSendReply(Response_);
PassAway();
}
@@ -317,11 +317,11 @@ protected:
Start();
if (Validate()) {
DoAction();
- } else {
+ } else {
SendReplyAndDie();
- }
- }
-
+ }
+ }
+
// Duration of request
virtual TDuration GetRequestDuration() const {
return FinishTs_ - StartTs_;
@@ -337,10 +337,10 @@ protected:
return GetRequestDuration() - GetRequestWaitDuration();
}
- virtual TString GetCustomACLPath() const {
- return GetQueuePath().GetQueuePath();
- }
-
+ virtual TString GetCustomACLPath() const {
+ return GetQueuePath().GetQueuePath();
+ }
+
virtual bool IsRequestSlow() const {
return GetRequestWorkingDuration() >= TDuration::MilliSeconds(Cfg().GetSlowRequestTimeMs());
}
@@ -349,57 +349,57 @@ protected:
RLOG_SQS_INFO("Request [" << UserName_ << "] [" << GetQueueName() << "] [" << Action_ << "] is slow. Working duration: " << GetRequestWorkingDuration().MilliSeconds() << "ms");
}
- TString SanitizeNodePath(const TString& path) const {
- TStringBuf sanitizedPath(path);
- // just skip SQS root path if there's such a prefix
+ TString SanitizeNodePath(const TString& path) const {
+ TStringBuf sanitizedPath(path);
+ // just skip SQS root path if there's such a prefix
if (sanitizedPath.SkipPrefix(TStringBuf(Cfg().GetRoot()))) { // always skip SQS root prefix
- return TString(sanitizedPath);
- } else {
- Y_VERIFY(false); // should never be applied in any other way
- }
-
- return {};
- }
-
- TString MakeAbsolutePath(const TString& relativePath) const {
- TStringBuilder fullPath;
+ return TString(sanitizedPath);
+ } else {
+ Y_VERIFY(false); // should never be applied in any other way
+ }
+
+ return {};
+ }
+
+ TString MakeAbsolutePath(const TString& relativePath) const {
+ TStringBuilder fullPath;
fullPath << Cfg().GetRoot();
- if (!relativePath.StartsWith("/")) {
- fullPath << "/";
- }
- fullPath << relativePath;
-
- return TString(fullPath);
- }
-
- size_t CalculatePathDepth(const TString& path) const {
- const TString sanitizedResource = TFsPath(path).Fix().GetPath();
- size_t count = 0;
- for (size_t i = 0, sz = sanitizedResource.size(); i < sz; ++i) {
- if (sanitizedResource[i] == '/') {
- ++count;
- }
- }
-
- return count;
- }
-
- bool IsCloud() const {
+ if (!relativePath.StartsWith("/")) {
+ fullPath << "/";
+ }
+ fullPath << relativePath;
+
+ return TString(fullPath);
+ }
+
+ size_t CalculatePathDepth(const TString& path) const {
+ const TString sanitizedResource = TFsPath(path).Fix().GetPath();
+ size_t count = 0;
+ for (size_t i = 0, sz = sanitizedResource.size(); i < sz; ++i) {
+ if (sanitizedResource[i] == '/') {
+ ++count;
+ }
+ }
+
+ return count;
+ }
+
+ bool IsCloud() const {
return Cfg().GetYandexCloudMode();
- }
-
- bool IsInternalResource(const TString& path) const {
- return CalculatePathDepth(SanitizeNodePath(path)) > 2;
- }
-
- bool IsForbiddenPath(const TString& path) const {
- return path.Contains("..") || path.Contains("//") || IsInternalResource(path);
- }
+ }
+
+ bool IsInternalResource(const TString& path) const {
+ return CalculatePathDepth(SanitizeNodePath(path)) > 2;
+ }
+
+ bool IsForbiddenPath(const TString& path) const {
+ return path.Contains("..") || path.Contains("//") || IsInternalResource(path);
+ }
struct TActionCountersPack {
TActionCounters* CoreCounters = nullptr;
TActionCounters* YmqCounters = nullptr;
};
-
+
TCountersCouple<TActionCounters*> GetActionCounters() const {
TCountersCouple<TActionCounters*> result{nullptr, nullptr};
if (IsActionForQueue(Action_) && QueueCounters_) {
@@ -424,16 +424,16 @@ protected:
}
void RequestSchemeCache(const TString& path) {
- auto schemeCacheRequest = MakeHolder<NSchemeCache::TSchemeCacheNavigate>();
- NSchemeCache::TSchemeCacheNavigate::TEntry entry;
-
- entry.Path = SplitPath(path);
- entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath;
- schemeCacheRequest->ResultSet.emplace_back(entry);
-
+ auto schemeCacheRequest = MakeHolder<NSchemeCache::TSchemeCacheNavigate>();
+ NSchemeCache::TSchemeCacheNavigate::TEntry entry;
+
+ entry.Path = SplitPath(path);
+ entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath;
+ schemeCacheRequest->ResultSet.emplace_back(entry);
+
this->Send(SchemeCache_, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release()));
- }
-
+ }
+
private:
STATEFN(InitialState) {
switch (ev->GetTypeRewrite()) {
@@ -444,13 +444,13 @@ private:
}
STATEFN(WaitAuthCheckMessages) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse);
hFunc(TEvTicketParser::TEvAuthorizeTicketResult, HandleTicketParserResponse);
hFunc(TEvWakeup, HandleWakeup);
- }
- }
-
+ }
+ }
+
STATEFN(WaitQuotaState) {
switch (ev->GetTypeRewrite()) {
hFunc(TEvQuota::TEvClearance, HandleQuota);
@@ -458,42 +458,42 @@ private:
}
}
- TString GetActionACLSourcePath() const {
+ TString GetActionACLSourcePath() const {
const EACLSourceType aclSourceType = GetActionACLSourceType(ToString(Action_));
- switch (aclSourceType) {
- case EACLSourceType::Unknown: {
- return {};
- }
- case EACLSourceType::RootDir: {
- return GetQueuePath().GetRootPath();
- }
- case EACLSourceType::AccountDir: {
- return GetQueuePath().GetUserPath();
- }
- case EACLSourceType::QueueDir: {
- return GetQueuePath().GetQueuePath();
- }
- case EACLSourceType::Custom: {
- return GetCustomACLPath();
- }
- }
-
- return {};
- }
-
+ switch (aclSourceType) {
+ case EACLSourceType::Unknown: {
+ return {};
+ }
+ case EACLSourceType::RootDir: {
+ return GetQueuePath().GetRootPath();
+ }
+ case EACLSourceType::AccountDir: {
+ return GetQueuePath().GetUserPath();
+ }
+ case EACLSourceType::QueueDir: {
+ return GetQueuePath().GetQueuePath();
+ }
+ case EACLSourceType::Custom: {
+ return GetCustomACLPath();
+ }
+ }
+
+ return {};
+ }
+
void RequestTicketParser() {
this->Send(MakeTicketParserID(), new TEvTicketParser::TEvAuthorizeTicket(SecurityToken_));
- }
-
- bool IsACLProtectedAccount(const TString& accountName) const {
- if (accountName) {
- // temporary O(N) solution since the list contains up to 100 items
+ }
+
+ bool IsACLProtectedAccount(const TString& accountName) const {
+ if (accountName) {
+ // temporary O(N) solution since the list contains up to 100 items
return !IsIn(Cfg().GetAccountsWithoutMandatoryAuth(), accountName);
- }
-
- return true;
- }
-
+ }
+
+ return true;
+ }
+
void HandleConfiguration(TSqsEvents::TEvConfiguration::TPtr& ev) {
const TDuration confDuration = TActivationContext::Now() - StartTs_;
RLOG_SQS_DEBUG("Get configuration duration: " << confDuration.MilliSeconds() << "ms");
@@ -502,17 +502,17 @@ private:
UserExists_ = ev->Get()->UserExists;
QueueExists_ = ev->Get()->QueueExists;
Shards_ = ev->Get()->Shards;
- IsFifo_ = ev->Get()->Fifo;
+ IsFifo_ = ev->Get()->Fifo;
QueueAttributes_ = std::move(ev->Get()->QueueAttributes);
- SchemeCache_ = ev->Get()->SchemeCache;
+ SchemeCache_ = ev->Get()->SchemeCache;
SqsCoreCounters_ = std::move(ev->Get()->SqsCoreCounters);
QueueCounters_ = std::move(ev->Get()->QueueCounters);
UserCounters_ = std::move(ev->Get()->UserCounters);
QueueLeader_ = ev->Get()->QueueLeader;
QuoterResources_ = std::move(ev->Get()->QuoterResources);
- Y_VERIFY(SchemeCache_);
-
+ Y_VERIFY(SchemeCache_);
+
RLOG_SQS_TRACE("Got configuration. Root url: " << RootUrl_
<< ", Shards: " << Shards_
<< ", Fail: " << ev->Get()->Fail);
@@ -548,15 +548,15 @@ private:
bool isACLProtectedAccount = Cfg().GetForceAccessControl();
if (!IsCloud() && (SecurityToken_ || (Cfg().GetForceAccessControl() && (isACLProtectedAccount = IsACLProtectedAccount(UserName_))))) {
- this->Become(&TActionActor::WaitAuthCheckMessages);
- const auto& actionACLSourcePath = GetActionACLSourcePath();
- if (!actionACLSourcePath || IsForbiddenPath(actionACLSourcePath)) {
+ this->Become(&TActionActor::WaitAuthCheckMessages);
+ const auto& actionACLSourcePath = GetActionACLSourcePath();
+ if (!actionACLSourcePath || IsForbiddenPath(actionACLSourcePath)) {
RLOG_SQS_ERROR("Bad ACL source path " << actionACLSourcePath << " for " << Action_ << " action");
- MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED);
+ MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED);
SendReplyAndDie();
- return;
- }
-
+ return;
+ }
+
if (!SecurityToken_) {
MakeError(MutableErrorDesc(), NErrors::INVALID_CLIENT_TOKEN_ID, "No security token was provided.");
SendReplyAndDie();
@@ -569,69 +569,69 @@ private:
if (!isACLProtectedAccount) { // !IsCloud && !SecurityToken_ && account is in AccountsWithoutMandatoryAuth setting.
INC_COUNTER(UserCounters_, UnauthenticatedAccess); // if !ForceAccessControl, this counter is not initialized.
}
- // old habits
+ // old habits
DoGetQuotaAndProcess();
- }
- }
-
+ }
+ }
+
void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get();
- const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get();
-
- Y_VERIFY(navigate->ResultSet.size() == 1);
-
- if (navigate->ErrorCount > 0) {
- const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status;
+ TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get();
+ const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get();
+
+ Y_VERIFY(navigate->ResultSet.size() == 1);
+
+ if (navigate->ErrorCount > 0) {
+ const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status;
RLOG_SQS_ERROR("Failed to read ACL for " << GetActionACLSourcePath() << ". Scheme cache error: " << status);
-
- if (status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) {
- MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED);
- } else {
- MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
- }
-
+
+ if (status == NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown) {
+ MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED);
+ } else {
+ MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
+ }
+
SendReplyAndDie();
- return;
+ return;
}
-
- SecurityObject_ = navigate->ResultSet.front().SecurityObject;
-
+
+ SecurityObject_ = navigate->ResultSet.front().SecurityObject;
+
OnAuthCheckMessage();
- }
-
+ }
+
void HandleTicketParserResponse(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev) {
const TEvTicketParser::TEvAuthorizeTicketResult& result(*ev->Get());
- if (!result.Error.empty()) {
+ if (!result.Error.empty()) {
RLOG_SQS_ERROR("Got ticket parser error: " << result.Error << ". " << Action_ << " was rejected");
- MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED);
+ MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED);
SendReplyAndDie();
- return;
- } else {
- UserToken_ = ev->Get()->Token;
- Y_VERIFY(UserToken_);
- }
-
+ return;
+ } else {
+ UserToken_ = ev->Get()->Token;
+ Y_VERIFY(UserToken_);
+ }
+
OnAuthCheckMessage();
}
void OnAuthCheckMessage() {
- --SecurityCheckRequestsToWaitFor_;
-
- if (SecurityCheckRequestsToWaitFor_ == 0) {
+ --SecurityCheckRequestsToWaitFor_;
+
+ if (SecurityCheckRequestsToWaitFor_ == 0) {
const TString& actionName = ToString(Action_);
const ui32 requiredAccess = GetActionRequiredAccess(actionName);
- UserSID_ = UserToken_->GetUserSID();
- if (requiredAccess != 0 && SecurityObject_ && !SecurityObject_->CheckAccess(requiredAccess, *UserToken_)) {
+ UserSID_ = UserToken_->GetUserSID();
+ if (requiredAccess != 0 && SecurityObject_ && !SecurityObject_->CheckAccess(requiredAccess, *UserToken_)) {
if (Action_ == EAction::ModifyPermissions) {
- // do not spam for other actions
+ // do not spam for other actions
RLOG_SQS_WARN("User " << UserSID_ << " tried to modify ACL for " << GetActionACLSourcePath() << ". Access denied");
- }
+ }
MakeError(MutableErrorDesc(), NErrors::ACCESS_DENIED, Sprintf("%s on %s was denied for %s due to missing permission %s.",
actionName.c_str(), SanitizeNodePath(GetActionACLSourcePath()).c_str(), UserSID_.c_str(), GetActionMatchingACE(actionName).c_str()));
SendReplyAndDie();
- return;
- }
-
+ return;
+ }
+
DoGetQuotaAndProcess();
}
}
@@ -662,9 +662,9 @@ private:
deadline));
} else {
DoRoutine();
- }
- }
-
+ }
+ }
+
void HandleQuota(TEvQuota::TEvClearance::TPtr& ev) {
const TDuration quotaWaitDuration = TActivationContext::Now() - QuotaRequestTs_;
switch (ev->Get()->Result) {
@@ -743,16 +743,16 @@ protected:
THolder<IReplyCallback> Cb_;
TString RootUrl_;
TString UserName_;
- TString SecurityToken_;
- TString FolderId_;
- size_t SecurityCheckRequestsToWaitFor_ = 2;
- TIntrusivePtr<TSecurityObject> SecurityObject_;
- TIntrusivePtr<NACLib::TUserToken> UserToken_;
- TString UserSID_; // identifies the client who sent this request
+ TString SecurityToken_;
+ TString FolderId_;
+ size_t SecurityCheckRequestsToWaitFor_ = 2;
+ TIntrusivePtr<TSecurityObject> SecurityObject_;
+ TIntrusivePtr<NACLib::TUserToken> UserToken_;
+ TString UserSID_; // identifies the client who sent this request
bool UserExists_ = false;
bool QueueExists_ = false;
ui64 Shards_;
- TMaybe<bool> IsFifo_;
+ TMaybe<bool> IsFifo_;
TInstant StartTs_;
TInstant FinishTs_;
TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCoreCounters_; // Raw counters interface. Is is not prefered to use them
diff --git a/ydb/core/ymq/actor/actor.cpp b/ydb/core/ymq/actor/actor.cpp
index cf7292a89a..d2ebd7cf50 100644
--- a/ydb/core/ymq/actor/actor.cpp
+++ b/ydb/core/ymq/actor/actor.cpp
@@ -52,21 +52,21 @@ IActor* CreateActionActor(const NKikimrClient::TSqsRequest& req, THolder<IReplyC
REQUEST_CASE(DeleteQueue)
REQUEST_CASE(DeleteQueueBatch)
REQUEST_CASE(DeleteUser)
- REQUEST_CASE(ListPermissions)
+ REQUEST_CASE(ListPermissions)
REQUEST_CASE(GetQueueAttributes)
REQUEST_CASE(GetQueueAttributesBatch)
REQUEST_CASE(GetQueueUrl)
REQUEST_CASE(ListQueues)
REQUEST_CASE(ListUsers)
- REQUEST_CASE(ModifyPermissions)
+ REQUEST_CASE(ModifyPermissions)
REQUEST_CASE(PurgeQueue)
REQUEST_CASE(PurgeQueueBatch)
REQUEST_CASE(ReceiveMessage)
REQUEST_CASE(SendMessage)
REQUEST_CASE(SendMessageBatch)
REQUEST_CASE(SetQueueAttributes)
- REQUEST_CASE(ListDeadLetterSourceQueues)
- REQUEST_CASE(CountQueues)
+ REQUEST_CASE(ListDeadLetterSourceQueues)
+ REQUEST_CASE(CountQueues)
#undef REQUEST_CASE
diff --git a/ydb/core/ymq/actor/count_queues.cpp b/ydb/core/ymq/actor/count_queues.cpp
index 2a47a9349a..8f217a192a 100644
--- a/ydb/core/ymq/actor/count_queues.cpp
+++ b/ydb/core/ymq/actor/count_queues.cpp
@@ -1,78 +1,78 @@
-#include "action.h"
-#include "error.h"
-#include "log.h"
-#include "params.h"
-#include "serviceid.h"
-#include "executor.h"
-
+#include "action.h"
+#include "error.h"
+#include "log.h"
+#include "params.h"
+#include "serviceid.h"
+#include "executor.h"
+
#include <ydb/public/lib/value/value.h>
-
-#include <util/string/ascii.h>
-#include <util/string/cast.h>
-#include <util/string/join.h>
-
-using NKikimr::NClient::TValue;
-
-namespace NKikimr::NSQS {
-
-class TCountQueuesActor
- : public TActionActor<TCountQueuesActor>
-{
-public:
- static constexpr bool NeedExistingQueue() {
- return false;
- }
-
+
+#include <util/string/ascii.h>
+#include <util/string/cast.h>
+#include <util/string/join.h>
+
+using NKikimr::NClient::TValue;
+
+namespace NKikimr::NSQS {
+
+class TCountQueuesActor
+ : public TActionActor<TCountQueuesActor>
+{
+public:
+ static constexpr bool NeedExistingQueue() {
+ return false;
+ }
+
TCountQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb)
: TActionActor(sourceSqsRequest, EAction::CountQueues, std::move(cb))
- {
+ {
CopyAccountName(Request());
- Response_.MutableCountQueues()->SetRequestId(RequestId_);
-
+ Response_.MutableCountQueues()->SetRequestId(RequestId_);
+
CopySecurityToken(Request());
- }
-
-private:
- TError* MutableErrorDesc() override {
- return Response_.MutableCountQueues()->MutableError();
- }
-
+ }
+
+private:
+ TError* MutableErrorDesc() override {
+ return Response_.MutableCountQueues()->MutableError();
+ }
+
void DoAction() override {
- Become(&TThis::StateFunc);
-
+ Become(&TThis::StateFunc);
+
Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvCountQueues(RequestId_, UserName_, FolderId_));
- }
-
- TString DoGetQueueName() const override {
- return TString();
- }
-
+ }
+
+ TString DoGetQueueName() const override {
+ return TString();
+ }
+
STATEFN(StateFunc) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TEvWakeup, HandleWakeup);
hFunc(TSqsEvents::TEvCountQueuesResponse, HandleCountQueuesResponse);
- }
- }
-
+ }
+ }
+
void HandleCountQueuesResponse(TSqsEvents::TEvCountQueuesResponse::TPtr& ev) {
- if (ev->Get()->Failed) {
+ if (ev->Get()->Failed) {
RLOG_SQS_WARN("Count queues failed");
- MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
- } else {
- auto* result = Response_.MutableCountQueues();
- result->SetCount(ev->Get()->Count);
- }
-
+ MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
+ } else {
+ auto* result = Response_.MutableCountQueues();
+ result->SetCount(ev->Get()->Count);
+ }
+
SendReplyAndDie();
- }
-
+ }
+
const TCountQueuesRequest& Request() const {
return SourceSqsRequest_.GetCountQueues();
}
-};
-
+};
+
IActor* CreateCountQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) {
return new TCountQueuesActor(sourceSqsRequest, std::move(cb));
-}
-
-} // namespace NKikimr::NSQS
+}
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/create_queue.cpp b/ydb/core/ymq/actor/create_queue.cpp
index 69c8e3078f..2d4b3af214 100644
--- a/ydb/core/ymq/actor/create_queue.cpp
+++ b/ydb/core/ymq/actor/create_queue.cpp
@@ -1,12 +1,12 @@
#include "action.h"
#include "error.h"
#include "log.h"
-#include "queue_schema.h"
+#include "queue_schema.h"
#include <ydb/core/ymq/base/constants.h>
#include <ydb/core/ymq/base/helpers.h>
#include <ydb/core/ymq/base/queue_id.h>
-
+
#include <util/string/join.h>
#include <util/string/type.h>
@@ -20,22 +20,22 @@ public:
return false;
}
- static constexpr bool CreateMissingAccount() {
- return true;
- }
-
+ static constexpr bool CreateMissingAccount() {
+ return true;
+ }
+
TCreateQueueActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb)
: TActionActor(sourceSqsRequest, EAction::CreateQueue, std::move(cb))
{
CopyAccountName(Request()); // will be replaced during bootstrap for cloud mode
Response_.MutableCreateQueue()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
-protected:
- bool IsFifoQueue() const override {
+protected:
+ bool IsFifoQueue() const override {
return AsciiHasSuffixIgnoreCase(Request().GetQueueName(), ".fifo"); // works for cloud too, since the custom name should end with '.fifo'
- }
+ }
private:
bool DoValidate() override {
@@ -104,19 +104,19 @@ private:
new TCreateQueueSchemaActorV2(TQueuePath(cfg.GetRoot(), accountName, queueName),
Request(), SelfId(), RequestId_, customQueueName, FolderId_, IsCloud(),
cfg.GetEnableQueueAttributesValidation(), UserCounters_, QuoterResources_)
- );
- }
-
+ );
+ }
+
void DoAction() override {
Become(&TThis::StateFunc);
- if (IsCloud()) {
+ if (IsCloud()) {
Register(new TAtomicCounterActor(SelfId(), Cfg().GetRoot(), RequestId_));
- } else {
- static const TString emptyCustomQueueName = "";
-
+ } else {
+ static const TString emptyCustomQueueName = "";
+
StartQueueCreation(Request().GetQueueName(), UserName_, emptyCustomQueueName);
- }
+ }
}
TString DoGetQueueName() const override {
@@ -132,25 +132,25 @@ private:
}
void HandleAtomicCounterIncrement(TSqsEvents::TEvAtomicCounterIncrementResult::TPtr& ev) {
- auto event = ev->Get();
+ auto event = ev->Get();
auto* result = Response_.MutableCreateQueue();
-
- if (event->Success) {
+
+ if (event->Success) {
const ui16 serviceId = Cfg().GetYandexCloudServiceId();
- const TString cloudId = UserName_; // should decode from creds
- ResourceId_ = MakeQueueId(serviceId, event->NewValue, UserName_);
+ const TString cloudId = UserName_; // should decode from creds
+ ResourceId_ = MakeQueueId(serviceId, event->NewValue, UserName_);
RLOG_SQS_DEBUG("Created resource id: " << MakeQueueId(serviceId, event->NewValue, UserName_)
- << " for service id: " << serviceId
- << " unique num: " << event->NewValue
- << " account name: " << UserName_);
-
+ << " for service id: " << serviceId
+ << " unique num: " << event->NewValue
+ << " account name: " << UserName_);
+
StartQueueCreation(ResourceId_, cloudId, Request().GetQueueName());
- } else {
- MakeError(result, NErrors::INTERNAL_FAILURE);
+ } else {
+ MakeError(result, NErrors::INTERNAL_FAILURE);
SendReplyAndDie();
- }
- }
-
+ }
+ }
+
void HandleQueueCreated(TSqsEvents::TEvQueueCreated::TPtr& ev) {
SchemaActor_ = TActorId();
auto event = ev->Get();
@@ -168,14 +168,14 @@ private:
case EQueueState::Active:
if (event->Success) {
const TString& name = Request().GetQueueName();
- if (IsCloud()) {
- const auto finalResourceId = event->AlreadyExists ? event->ExistingQueueResourceId : ResourceId_;
- result->SetQueueName(finalResourceId);
- result->SetQueueUrl(MakeQueueUrl(TString::Join(finalResourceId, '/', name)));
- } else {
- result->SetQueueName(name);
- result->SetQueueUrl(MakeQueueUrl(name));
- }
+ if (IsCloud()) {
+ const auto finalResourceId = event->AlreadyExists ? event->ExistingQueueResourceId : ResourceId_;
+ result->SetQueueName(finalResourceId);
+ result->SetQueueUrl(MakeQueueUrl(TString::Join(finalResourceId, '/', name)));
+ } else {
+ result->SetQueueName(name);
+ result->SetQueueUrl(MakeQueueUrl(name));
+ }
} else {
MakeError(result, *ev->Get()->ErrorClass, errMsg);
}
@@ -202,7 +202,7 @@ private:
}
private:
- TString ResourceId_;
+ TString ResourceId_;
TActorId SchemaActor_;
};
diff --git a/ydb/core/ymq/actor/create_user.cpp b/ydb/core/ymq/actor/create_user.cpp
index f8386a7609..deebfc54e8 100644
--- a/ydb/core/ymq/actor/create_user.cpp
+++ b/ydb/core/ymq/actor/create_user.cpp
@@ -19,7 +19,7 @@ public:
{
CopyAccountName(Request());
Response_.MutableCreateUser()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
diff --git a/ydb/core/ymq/actor/delete_message.cpp b/ydb/core/ymq/actor/delete_message.cpp
index ec923b4992..50b59ddcfb 100644
--- a/ydb/core/ymq/actor/delete_message.cpp
+++ b/ydb/core/ymq/actor/delete_message.cpp
@@ -103,18 +103,18 @@ private:
void ProcessAnswer(TDeleteMessageResponse* resp, const TSqsEvents::TEvDeleteMessageBatchResponse::TMessageResult& answer) {
switch (answer.Status) {
- case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::OK: {
+ case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::OK: {
INC_COUNTER_COUPLE(QueueCounters_, DeleteMessage_Count, deleted_count_per_second);
- break;
- }
- case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::NotFound: {
- // ignore missing handle just like proper SQS does
- break;
- }
- case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::Failed: {
- MakeError(resp, NErrors::INTERNAL_FAILURE);
- break;
- }
+ break;
+ }
+ case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::NotFound: {
+ // ignore missing handle just like proper SQS does
+ break;
+ }
+ case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::Failed: {
+ MakeError(resp, NErrors::INTERNAL_FAILURE);
+ break;
+ }
}
}
diff --git a/ydb/core/ymq/actor/delete_queue.cpp b/ydb/core/ymq/actor/delete_queue.cpp
index af76041aae..9711b91734 100644
--- a/ydb/core/ymq/actor/delete_queue.cpp
+++ b/ydb/core/ymq/actor/delete_queue.cpp
@@ -1,7 +1,7 @@
#include "action.h"
#include "common_batch_actor.h"
#include "error.h"
-#include "queue_schema.h"
+#include "queue_schema.h"
#include <util/string/join.h>
@@ -18,8 +18,8 @@ public:
Response_.MutableDeleteQueue()->SetRequestId(RequestId_);
CopySecurityToken(Request());
- }
-
+ }
+
private:
bool DoValidate() override {
if (!GetQueueName()) {
@@ -38,9 +38,9 @@ private:
Become(&TThis::StateFunc);
SchemaActor_ = Register(
- new TDeleteQueueSchemaActorV2(
+ new TDeleteQueueSchemaActorV2(
TQueuePath(Cfg().GetRoot(), UserName_, GetQueueName()), SelfId(), RequestId_, UserCounters_)
- );
+ );
}
TString DoGetQueueName() const override {
@@ -101,11 +101,11 @@ private:
const auto& entry = Request().GetEntries(i);
auto& req = *ret[i].MutableDeleteQueue();
req.MutableAuth()->SetUserName(UserName_);
-
+
if (Request().HasCredentials()) {
*req.MutableCredentials() = Request().GetCredentials();
- }
-
+ }
+
req.SetQueueName(entry.GetQueueName());
req.SetId(entry.GetId());
}
diff --git a/ydb/core/ymq/actor/delete_user.cpp b/ydb/core/ymq/actor/delete_user.cpp
index 0a3d230265..bea58c5a7a 100644
--- a/ydb/core/ymq/actor/delete_user.cpp
+++ b/ydb/core/ymq/actor/delete_user.cpp
@@ -2,7 +2,7 @@
#include "error.h"
#include "executor.h"
#include "log.h"
-#include "queue_schema.h"
+#include "queue_schema.h"
#include <ydb/public/lib/value/value.h>
@@ -25,7 +25,7 @@ public:
{
CopyAccountName(Request());
Response_.MutableDeleteUser()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
@@ -51,8 +51,8 @@ private:
.QueryId(LIST_QUEUES_ID)
.Counters(QueueCounters_)
.RetryOnTimeout()
- .Params()
- .Utf8("FOLDERID", "")
+ .Params()
+ .Utf8("FOLDERID", "")
.Utf8("USER_NAME", UserName_)
.ParentBuilder().Start();
}
@@ -94,7 +94,7 @@ private:
Queues_.insert(name);
Register(
- new TDeleteQueueSchemaActorV2(
+ new TDeleteQueueSchemaActorV2(
TQueuePath(Cfg().GetRoot(), Request().GetUserName(), name), SelfId(), RequestId_, UserCounters_)
);
}
diff --git a/ydb/core/ymq/actor/error.cpp b/ydb/core/ymq/actor/error.cpp
index b10337579e..23a65c56f6 100644
--- a/ydb/core/ymq/actor/error.cpp
+++ b/ydb/core/ymq/actor/error.cpp
@@ -62,7 +62,7 @@ size_t ErrorsCount(const NKikimrClient::TSqsResponse& response, TAPIStatusesCoun
RESPONSE_CASE(DeleteQueue)
RESPONSE_BATCH_CASE(DeleteQueueBatch)
RESPONSE_CASE(DeleteUser)
- RESPONSE_CASE(ListPermissions)
+ RESPONSE_CASE(ListPermissions)
RESPONSE_CASE(GetQueueAttributes)
RESPONSE_BATCH_CASE(GetQueueAttributesBatch)
RESPONSE_CASE(GetQueueUrl)
@@ -75,8 +75,8 @@ size_t ErrorsCount(const NKikimrClient::TSqsResponse& response, TAPIStatusesCoun
RESPONSE_CASE(SendMessage)
RESPONSE_BATCH_CASE(SendMessageBatch)
RESPONSE_CASE(SetQueueAttributes)
- RESPONSE_CASE(ListDeadLetterSourceQueues)
- RESPONSE_CASE(CountQueues)
+ RESPONSE_CASE(ListDeadLetterSourceQueues)
+ RESPONSE_CASE(CountQueues)
case NKikimrClient::TSqsResponse::RESPONSE_NOT_SET:
return 0;
diff --git a/ydb/core/ymq/actor/events.h b/ydb/core/ymq/actor/events.h
index 5cb94b5aba..9fff62c7cf 100644
--- a/ydb/core/ymq/actor/events.h
+++ b/ydb/core/ymq/actor/events.h
@@ -1,6 +1,6 @@
#pragma once
#include "defs.h"
-
+
#include <ydb/core/base/defs.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/core/tx/tx_proxy/proxy.h>
@@ -20,7 +20,7 @@
#include <library/cpp/actors/core/event_local.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
-#include <util/generic/hash.h>
+#include <util/generic/hash.h>
#include <util/generic/maybe.h>
#include <util/generic/ptr.h>
@@ -69,7 +69,7 @@ struct TSqsEvents {
/// Update queue attributes cache
EvClearQueueAttributesCache,
/// Incrementing of atomic counter
- EvAtomicCounterIncrementResult,
+ EvAtomicCounterIncrementResult,
/// Request for finding leader node for the given queue
EvGetLeaderNodeForQueueRequest,
@@ -81,10 +81,10 @@ struct TSqsEvents {
EvQueueId,
// Cloud specific
- EvGetQueueFolderIdAndCustomName,
- EvQueueFolderIdAndCustomName,
- EvCountQueues,
- EvCountQueuesResponse,
+ EvGetQueueFolderIdAndCustomName,
+ EvQueueFolderIdAndCustomName,
+ EvCountQueues,
+ EvCountQueuesResponse,
// Send/Receive/Delete requests. Action actor sends these requests to queue leader
EvSendMessageBatch,
@@ -107,8 +107,8 @@ struct TSqsEvents {
EvMigrationDone,
- EvReportProcessedRequestAttributes,
-
+ EvReportProcessedRequestAttributes,
+
EvInsertQueueCounters,
EvUserSettingsChanged,
@@ -116,14 +116,14 @@ struct TSqsEvents {
EvReadQueuesList,
EvQueuesList,
- EvDeadLetterQueueNotification,
-
- EvSchemeTraversalResult,
-
- EvGarbageCleaningResult,
-
- EvGarbageSearchResult,
-
+ EvDeadLetterQueueNotification,
+
+ EvSchemeTraversalResult,
+
+ EvGarbageCleaningResult,
+
+ EvGarbageSearchResult,
+
EvCleanupQueryComplete,
EvEnd,
@@ -350,21 +350,21 @@ struct TSqsEvents {
}
};
- struct TEvAtomicCounterIncrementResult : public NActors::TEventLocal<TEvAtomicCounterIncrementResult, EvAtomicCounterIncrementResult> {
- bool Success;
- TString Error;
- ui64 NewValue;
-
- TEvAtomicCounterIncrementResult(bool success, const TString& error = TString(), ui64 newValue = 0)
- : Success(success)
- , Error(error)
- , NewValue(newValue)
- {
- }
-
- TEvAtomicCounterIncrementResult(const TEvAtomicCounterIncrementResult& other) = default;
- };
-
+ struct TEvAtomicCounterIncrementResult : public NActors::TEventLocal<TEvAtomicCounterIncrementResult, EvAtomicCounterIncrementResult> {
+ bool Success;
+ TString Error;
+ ui64 NewValue;
+
+ TEvAtomicCounterIncrementResult(bool success, const TString& error = TString(), ui64 newValue = 0)
+ : Success(success)
+ , Error(error)
+ , NewValue(newValue)
+ {
+ }
+
+ TEvAtomicCounterIncrementResult(const TEvAtomicCounterIncrementResult& other) = default;
+ };
+
struct TEvPurgeQueue : public NActors::TEventLocal<TEvPurgeQueue, EvPurgeQueue> {
/// Queue path in the catalog
TQueuePath QueuePath;
@@ -516,29 +516,29 @@ struct TSqsEvents {
bool Exists = false;
bool Failed = false;
TString QueueId; // resource id in case of Yandex.Cloud mode and queue name in case of Yandex
- ui64 Version = 0; // last queue version registered in service actor
- ui64 ShardsCount = 0; // number of queue shards
+ ui64 Version = 0; // last queue version registered in service actor
+ ui64 ShardsCount = 0; // number of queue shards
- TEvQueueId(const bool failed = false)
+ TEvQueueId(const bool failed = false)
: Failed(failed)
{
}
- explicit TEvQueueId(const TString queueId, const ui64 version, const ui64 shardsCount)
+ explicit TEvQueueId(const TString queueId, const ui64 version, const ui64 shardsCount)
: Exists(true)
, QueueId(std::move(queueId))
- , Version(version)
- , ShardsCount(shardsCount)
+ , Version(version)
+ , ShardsCount(shardsCount)
{
}
};
- struct TEvGetQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvGetQueueFolderIdAndCustomName, EvGetQueueFolderIdAndCustomName> {
+ struct TEvGetQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvGetQueueFolderIdAndCustomName, EvGetQueueFolderIdAndCustomName> {
TString RequestId;
TString UserName;
TString QueueName;
- TEvGetQueueFolderIdAndCustomName(TString requestId, TString userName, TString queueName)
+ TEvGetQueueFolderIdAndCustomName(TString requestId, TString userName, TString queueName)
: RequestId(std::move(requestId))
, UserName(std::move(userName))
, QueueName(std::move(queueName))
@@ -546,51 +546,51 @@ struct TSqsEvents {
}
};
- struct TEvQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvQueueFolderIdAndCustomName, EvQueueFolderIdAndCustomName> {
+ struct TEvQueueFolderIdAndCustomName : public NActors::TEventLocal<TEvQueueFolderIdAndCustomName, EvQueueFolderIdAndCustomName> {
bool Exists = false;
bool Failed = false;
TString QueueFolderId;
- TString QueueCustomName;
+ TString QueueCustomName;
- TEvQueueFolderIdAndCustomName(bool failed = false)
+ TEvQueueFolderIdAndCustomName(bool failed = false)
: Failed(failed)
{
}
- explicit TEvQueueFolderIdAndCustomName(TString queueFolderId, TString queueCustomName)
+ explicit TEvQueueFolderIdAndCustomName(TString queueFolderId, TString queueCustomName)
: Exists(true)
, QueueFolderId(std::move(queueFolderId))
- , QueueCustomName(std::move(queueCustomName))
+ , QueueCustomName(std::move(queueCustomName))
+ {
+ }
+ };
+
+ struct TEvCountQueues : public NActors::TEventLocal<TEvCountQueues, EvCountQueues> {
+ TString RequestId;
+ TString UserName;
+ TString FolderId;
+
+ TEvCountQueues(TString requestId, TString userName, TString folderId)
+ : RequestId(std::move(requestId))
+ , UserName(std::move(userName))
+ , FolderId(std::move(folderId))
+ {
+ }
+ };
+
+ struct TEvCountQueuesResponse : public NActors::TEventLocal<TEvCountQueuesResponse, EvCountQueuesResponse> {
+ bool Failed = false;
+ bool Exists = false;
+ ui64 Count = 0;
+
+ explicit TEvCountQueuesResponse(bool failed, bool exists = false, ui64 count = 0)
+ : Failed(failed)
+ , Exists(exists)
+ , Count(count)
{
}
};
- struct TEvCountQueues : public NActors::TEventLocal<TEvCountQueues, EvCountQueues> {
- TString RequestId;
- TString UserName;
- TString FolderId;
-
- TEvCountQueues(TString requestId, TString userName, TString folderId)
- : RequestId(std::move(requestId))
- , UserName(std::move(userName))
- , FolderId(std::move(folderId))
- {
- }
- };
-
- struct TEvCountQueuesResponse : public NActors::TEventLocal<TEvCountQueuesResponse, EvCountQueuesResponse> {
- bool Failed = false;
- bool Exists = false;
- ui64 Count = 0;
-
- explicit TEvCountQueuesResponse(bool failed, bool exists = false, ui64 count = 0)
- : Failed(failed)
- , Exists(exists)
- , Count(count)
- {
- }
- };
-
struct TEvSendMessageBatch : public NActors::TEventLocal<TEvSendMessageBatch, EvSendMessageBatch> {
struct TMessageEntry {
TString MessageId;
@@ -750,10 +750,10 @@ struct TSqsEvents {
{
}
};
-
- struct TEvReportProcessedRequestAttributes : public NActors::TEventLocal<TEvReportProcessedRequestAttributes, EvReportProcessedRequestAttributes> {
- TProcessedRequestAttributes Data;
- };
+
+ struct TEvReportProcessedRequestAttributes : public NActors::TEventLocal<TEvReportProcessedRequestAttributes, EvReportProcessedRequestAttributes> {
+ TProcessedRequestAttributes Data;
+ };
struct TEvInsertQueueCounters : public NActors::TEventLocal<TEvInsertQueueCounters, EvInsertQueueCounters> {
TEvInsertQueueCounters(const TString& user, const TString& queue, ui64 leaderTabletId)
@@ -792,7 +792,7 @@ struct TSqsEvents {
ui64 LeaderTabletId = 0;
TString CustomName;
TString FolderId;
- TString DlqName;
+ TString DlqName;
ui64 Version = 0;
ui64 ShardsCount = 0;
TInstant CreatedTimestamp;
@@ -810,72 +810,72 @@ struct TSqsEvents {
{
}
};
-
+
// Used by service to notify dead letter queue leader
- struct TEvDeadLetterQueueNotification : public NActors::TEventLocal<TEvDeadLetterQueueNotification, EvDeadLetterQueueNotification> {
- };
-
- using TSchemePath = TVector<TString>;
- using TSchemeCacheNavigate = NSchemeCache::TSchemeCacheNavigate;
-
- struct TSchemeNode {
- TSchemePath Path;
- TSchemeCacheNavigate::EKind Kind = TSchemeCacheNavigate::EKind::KindUnknown;
- TInstant CreationTs = TInstant::Zero();
-
- THashMap<TString, TSchemeNode> Children;
- };
-
- struct TEvSchemeTraversalResult : public NActors::TEventLocal<TEvSchemeTraversalResult, EvSchemeTraversalResult> {
- explicit TEvSchemeTraversalResult(bool success = true)
- : Success(success)
- {
- }
-
- bool Success = true;
- THolder<TSchemeNode> RootHolder;
- };
-
- struct TEvGarbageCleaningResult : public NActors::TEventLocal<TEvGarbageCleaningResult, EvGarbageCleaningResult> {
- explicit TEvGarbageCleaningResult(bool success = true) {
- Record.Success = success;
- }
-
- struct TCleaningResult {
- bool Success = true;
-
- TString Account;
- TString HintPath;
- TVector<TString> RemovedNodes;
-
- TInstant StartedAt;
- TDuration Duration;
- };
-
- TCleaningResult Record;
- };
-
- struct TEvGarbageSearchResult : public NActors::TEventLocal<TEvGarbageSearchResult, EvGarbageSearchResult> {
- explicit TEvGarbageSearchResult(bool success = true)
- : Success(success)
- {
- }
-
- struct TGarbageHint {
- enum class EReason {
- Unknown,
- UnusedVersion,
- UnregisteredQueue
- };
-
- TString Account;
- TSchemeNode SchemeNode;
- EReason Reason = EReason::Unknown;
- };
-
- bool Success = true;
- THashMap<TString, TGarbageHint> GarbageHints;
- };
+ struct TEvDeadLetterQueueNotification : public NActors::TEventLocal<TEvDeadLetterQueueNotification, EvDeadLetterQueueNotification> {
+ };
+
+ using TSchemePath = TVector<TString>;
+ using TSchemeCacheNavigate = NSchemeCache::TSchemeCacheNavigate;
+
+ struct TSchemeNode {
+ TSchemePath Path;
+ TSchemeCacheNavigate::EKind Kind = TSchemeCacheNavigate::EKind::KindUnknown;
+ TInstant CreationTs = TInstant::Zero();
+
+ THashMap<TString, TSchemeNode> Children;
+ };
+
+ struct TEvSchemeTraversalResult : public NActors::TEventLocal<TEvSchemeTraversalResult, EvSchemeTraversalResult> {
+ explicit TEvSchemeTraversalResult(bool success = true)
+ : Success(success)
+ {
+ }
+
+ bool Success = true;
+ THolder<TSchemeNode> RootHolder;
+ };
+
+ struct TEvGarbageCleaningResult : public NActors::TEventLocal<TEvGarbageCleaningResult, EvGarbageCleaningResult> {
+ explicit TEvGarbageCleaningResult(bool success = true) {
+ Record.Success = success;
+ }
+
+ struct TCleaningResult {
+ bool Success = true;
+
+ TString Account;
+ TString HintPath;
+ TVector<TString> RemovedNodes;
+
+ TInstant StartedAt;
+ TDuration Duration;
+ };
+
+ TCleaningResult Record;
+ };
+
+ struct TEvGarbageSearchResult : public NActors::TEventLocal<TEvGarbageSearchResult, EvGarbageSearchResult> {
+ explicit TEvGarbageSearchResult(bool success = true)
+ : Success(success)
+ {
+ }
+
+ struct TGarbageHint {
+ enum class EReason {
+ Unknown,
+ UnusedVersion,
+ UnregisteredQueue
+ };
+
+ TString Account;
+ TSchemeNode SchemeNode;
+ EReason Reason = EReason::Unknown;
+ };
+
+ bool Success = true;
+ THashMap<TString, TGarbageHint> GarbageHints;
+ };
struct TEvCleanupQueryComplete : public NActors::TEventLocal<TEvCleanupQueryComplete, EvCleanupQueryComplete> {
explicit TEvCleanupQueryComplete(const TString& name, ui64 type)
: Name(name)
diff --git a/ydb/core/ymq/actor/executor.cpp b/ydb/core/ymq/actor/executor.cpp
index 3c60e38b7d..e17dc68fe9 100644
--- a/ydb/core/ymq/actor/executor.cpp
+++ b/ydb/core/ymq/actor/executor.cpp
@@ -4,7 +4,7 @@
#include <ydb/core/protos/tx_proxy.pb.h>
#include <ydb/core/protos/flat_tx_scheme.pb.h>
-
+
#include <ydb/core/engine/mkql_proto.h>
#include <ydb/public/lib/value/value.h>
#include <ydb/core/ymq/base/debug_info.h>
@@ -109,10 +109,10 @@ void TExecutorBuilder::SendToQueueLeader() {
TActivationContext::Send(new IEventHandle(QueueLeaderActor_, Parent_, ev.Release()));
}
-const char* TExecutorBuilder::GetQueryById(size_t idx) {
- const char* query = IsFifoQueue_ ? GetFifoQueryById(idx) : GetStdQueryById(idx);
- Y_VERIFY(query);
- return query;
+const char* TExecutorBuilder::GetQueryById(size_t idx) {
+ const char* query = IsFifoQueue_ ? GetFifoQueryById(idx) : GetStdQueryById(idx);
+ Y_VERIFY(query);
+ return query;
}
TMiniKqlExecutionActor::TMiniKqlExecutionActor(
@@ -274,19 +274,19 @@ void TMiniKqlExecutionActor::HandleCompile(TMiniKQLCompileServiceEvents::TEvComp
ProceedWithExecution();
}
-template<typename TKikimrResultRecord>
-bool TMiniKqlExecutionActor::ShouldRetryOnFail(const TKikimrResultRecord& record) const {
- const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus());
- return NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(status) ||
- RetryOnTimeout_ && record.GetStatusCode() == NKikimrIssues::TStatusIds::TIMEOUT ||
+template<typename TKikimrResultRecord>
+bool TMiniKqlExecutionActor::ShouldRetryOnFail(const TKikimrResultRecord& record) const {
+ const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus());
+ return NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(status) ||
+ RetryOnTimeout_ && record.GetStatusCode() == NKikimrIssues::TStatusIds::TIMEOUT ||
(record.HasSchemeShardStatus() && record.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusMultipleModifications); // very rare case in queue creation
-}
-
+}
+
void TMiniKqlExecutionActor::HandleResponse(TResponse::TPtr& ev) {
const TDuration executionDuration = TActivationContext::Now() - StartExecutionTs_;
auto& response = *ev->Get();
- auto& record = response.Record;
- const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus());
+ auto& record = response.Record;
+ const auto status = NKikimr::NTxProxy::TResultStatus::EStatus(record.GetStatus());
RLOG_SQS_TRACE(GetRequestType() << " Queue " << TLogQueueName(QueuePath_) << " HandleResponse " << record);
RLOG_SQS_DEBUG(GetRequestType() << " Queue " << TLogQueueName(QueuePath_) << " Attempt " << AttemptNumber_ << " execution duration: " << executionDuration.MilliSeconds() << "ms");
if (Counters_) {
@@ -314,7 +314,7 @@ void TMiniKqlExecutionActor::HandleResponse(TResponse::TPtr& ev) {
&& status != TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress
&& status != TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecAlready) {
failed = true;
- retryableError = ShouldRetryOnFail(record);
+ retryableError = ShouldRetryOnFail(record);
if (retryableError) {
RLOG_SQS_WARN(GetRequestType() << " Queue " << TLogQueueName(QueuePath_) << " Retryable error in mkql execution result: " << response.Record);
} else {
diff --git a/ydb/core/ymq/actor/executor.h b/ydb/core/ymq/actor/executor.h
index 30d142b4ed..5dd51c6da2 100644
--- a/ydb/core/ymq/actor/executor.h
+++ b/ydb/core/ymq/actor/executor.h
@@ -36,10 +36,10 @@ public:
Shard_ = shard;
return *this;
}
- TExecutorBuilder& QueueVersion(ui64 version) {
- QueueVersion_ = version;
- return *this;
- }
+ TExecutorBuilder& QueueVersion(ui64 version) {
+ QueueVersion_ = version;
+ return *this;
+ }
TExecutorBuilder& QueueLeader(const TActorId& queueLeaderActor) {
QueueLeaderActor_ = queueLeaderActor;
return *this;
@@ -60,10 +60,10 @@ public:
QueryId_ = id;
return *this;
}
- TExecutorBuilder& Fifo(bool isFifo) {
- IsFifoQueue_ = isFifo;
- return *this;
- }
+ TExecutorBuilder& Fifo(bool isFifo) {
+ IsFifoQueue_ = isFifo;
+ return *this;
+ }
TExecutorBuilder& Mode(NKikimrTxUserProxy::TMiniKQLTransaction::EMode mode) {
Request().Record.MutableTransaction()->MutableMiniKQLTransaction()->SetMode(mode);
return *this;
@@ -126,7 +126,7 @@ private:
return QueryId_ != EQueryId::QUERY_VECTOR_SIZE;
}
- const char* GetQueryById(size_t idx);
+ const char* GetQueryById(size_t idx);
private:
const TActorId Parent_;
@@ -134,11 +134,11 @@ private:
THolder<TEvTxUserProxy::TEvProposeTransaction> ProposeTransactionRequest_;
TMaybe<TParameters> Parameters_;
bool RetryOnTimeout_ = false;
- bool IsFifoQueue_ = false;
+ bool IsFifoQueue_ = false;
TString UserName_;
TString QueueName_;
ui64 Shard_ = 0;
- ui64 QueueVersion_ = 0;
+ ui64 QueueVersion_ = 0;
TActorId QueueLeaderActor_;
TSqsEvents::TExecutedCallback Callback_;
EQueryId QueryId_ = EQueryId::QUERY_VECTOR_SIZE;
@@ -197,10 +197,10 @@ private:
void PassAway();
void HandleCompile(TMiniKQLCompileServiceEvents::TEvCompileStatus::TPtr& ev);
-
- template<typename TKikimrResultRecord>
+
+ template<typename TKikimrResultRecord>
bool ShouldRetryOnFail(const TKikimrResultRecord& record) const;
-
+
void HandleResponse(TResponse::TPtr& ev);
void HandleWakeup(TEvWakeup::TPtr& ev);
void HandleResult(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev);
diff --git a/ydb/core/ymq/actor/garbage_collector.cpp b/ydb/core/ymq/actor/garbage_collector.cpp
index c67a72dc36..e82284c020 100644
--- a/ydb/core/ymq/actor/garbage_collector.cpp
+++ b/ydb/core/ymq/actor/garbage_collector.cpp
@@ -1,796 +1,796 @@
-#include "garbage_collector.h"
-
-#include "cfg.h"
-#include "log.h"
-#include "events.h"
-#include "executor.h"
-#include "schema.h"
-
+#include "garbage_collector.h"
+
+#include "cfg.h"
+#include "log.h"
+#include "events.h"
+#include "executor.h"
+#include "schema.h"
+
#include <ydb/core/base/path.h>
#include <ydb/core/mon/mon.h>
-
+
#include <ydb/core/protos/services.pb.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
-
+
#include <library/cpp/monlib/service/pages/templates.h>
-
-#include <util/string/join.h>
-
-#include <algorithm>
-#include <queue>
-#include <vector>
-
-namespace NKikimr::NSQS {
-
-using TQueueRecord = TSqsEvents::TEvQueuesList::TQueueRecord;
-
-using TSchemePath = TSqsEvents::TSchemePath;
-using TSchemeNode = TSqsEvents::TSchemeNode;
-using TGarbageHint = TSqsEvents::TEvGarbageSearchResult::TGarbageHint;
-using TSchemeCacheNavigate = TSqsEvents::TSchemeCacheNavigate;
-using TCleaningResult = TSqsEvents::TEvGarbageCleaningResult::TCleaningResult;
-
-static const TString GARBAGE_CLEANER_LABEL = "GarbageCleaner";
-static const TString MINIMUM_ITEM_AGE_SECONDS = "minimum_item_age_seconds";
-static const TString PATH_TO_CLEAN = "path_to_clean";
-
-static void PrintAllPaths(const TSchemeNode& node, TStringStream& ss) {
- if (node.Path.size()) {
- ss << CanonizePath(node.Path) << ", kind: " << static_cast<size_t>(node.Kind) << ", ts: " << node.CreationTs << "\n";
- }
-
- for (const auto& [name, child] : node.Children) {
- PrintAllPaths(child, ss);
- }
-}
-
-static TSchemeNode& FindOrAllocateSchemeNode(TSchemeNode* root, const TSchemePath& path) {
- TSchemeNode* currentNode = root;
-
- for (const auto& part : path) {
- currentNode = &currentNode->Children[part];
- }
-
- return *currentNode;
-}
-
-class TSchemeTraversalActor : public TActorBootstrapped<TSchemeTraversalActor> {
-public:
+
+#include <util/string/join.h>
+
+#include <algorithm>
+#include <queue>
+#include <vector>
+
+namespace NKikimr::NSQS {
+
+using TQueueRecord = TSqsEvents::TEvQueuesList::TQueueRecord;
+
+using TSchemePath = TSqsEvents::TSchemePath;
+using TSchemeNode = TSqsEvents::TSchemeNode;
+using TGarbageHint = TSqsEvents::TEvGarbageSearchResult::TGarbageHint;
+using TSchemeCacheNavigate = TSqsEvents::TSchemeCacheNavigate;
+using TCleaningResult = TSqsEvents::TEvGarbageCleaningResult::TCleaningResult;
+
+static const TString GARBAGE_CLEANER_LABEL = "GarbageCleaner";
+static const TString MINIMUM_ITEM_AGE_SECONDS = "minimum_item_age_seconds";
+static const TString PATH_TO_CLEAN = "path_to_clean";
+
+static void PrintAllPaths(const TSchemeNode& node, TStringStream& ss) {
+ if (node.Path.size()) {
+ ss << CanonizePath(node.Path) << ", kind: " << static_cast<size_t>(node.Kind) << ", ts: " << node.CreationTs << "\n";
+ }
+
+ for (const auto& [name, child] : node.Children) {
+ PrintAllPaths(child, ss);
+ }
+}
+
+static TSchemeNode& FindOrAllocateSchemeNode(TSchemeNode* root, const TSchemePath& path) {
+ TSchemeNode* currentNode = root;
+
+ for (const auto& part : path) {
+ currentNode = &currentNode->Children[part];
+ }
+
+ return *currentNode;
+}
+
+class TSchemeTraversalActor : public TActorBootstrapped<TSchemeTraversalActor> {
+public:
TSchemeTraversalActor(const TActorId parentId,
const TActorId schemeCacheId,
- const TVector<TSchemePath>& pathsToTraverse,
- const ui64 maxDepth = Max())
- : ParentId(parentId)
- , SchemeCacheId(schemeCacheId)
- , PathsToTraverse(pathsToTraverse)
- , MaxDepth(maxDepth)
- , CurrentDepth(0)
- {
- }
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
- }
-
- void Bootstrap() {
- Become(&TThis::Working);
-
- RootHolder = MakeHolder<TSchemeNode>();
-
- SendListChildrenRequest(PathsToTraverse);
- }
-
-private:
- void SendListChildrenRequest(const TVector<TSchemePath>& paths) {
- const size_t elementsCount = paths.size();
-
- Y_VERIFY(elementsCount);
-
- auto schemeCacheRequest = MakeHolder<TSchemeCacheNavigate>();
-
- schemeCacheRequest->ResultSet.resize(elementsCount);
-
- for (size_t i = 0; i < elementsCount; ++i) {
- schemeCacheRequest->ResultSet[i].Path = paths[i];
-
- schemeCacheRequest->ResultSet[i].Operation =
- ((CurrentDepth == MaxDepth) ? TSchemeCacheNavigate::OpPath : TSchemeCacheNavigate::OpList);
- }
-
- Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release()));
- }
-
- STATEFN(Working) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse);
- }
- }
-
- void OnFinishedTraversal(bool success) {
- auto ev = MakeHolder<TSqsEvents::TEvSchemeTraversalResult>(success);
-
- if (success) {
- ev->RootHolder = std::move(RootHolder);
- }
-
- Send(ParentId, ev.Release());
-
- PassAway();
- }
-
- void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
- const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get();
-
- if (navigate->ErrorCount) {
- OnFinishedTraversal(false);
- return;
- }
-
- TVector<TSchemePath> newPathsToTraverse;
-
- ++CurrentDepth;
-
- for (const auto& entry : navigate->ResultSet) {
- auto& node = FindOrAllocateSchemeNode(&*RootHolder, entry.Path);
-
- node.Path = entry.Path;
- node.Kind = entry.Kind;
- node.CreationTs = TInstant::MilliSeconds(entry.CreateStep);
-
- if (entry.ListNodeEntry && CurrentDepth <= MaxDepth) {
- for (const auto& child : entry.ListNodeEntry->Children) {
- auto yetAnotherPathToTraverse = entry.Path;
+ const TVector<TSchemePath>& pathsToTraverse,
+ const ui64 maxDepth = Max())
+ : ParentId(parentId)
+ , SchemeCacheId(schemeCacheId)
+ , PathsToTraverse(pathsToTraverse)
+ , MaxDepth(maxDepth)
+ , CurrentDepth(0)
+ {
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
+ }
+
+ void Bootstrap() {
+ Become(&TThis::Working);
+
+ RootHolder = MakeHolder<TSchemeNode>();
+
+ SendListChildrenRequest(PathsToTraverse);
+ }
+
+private:
+ void SendListChildrenRequest(const TVector<TSchemePath>& paths) {
+ const size_t elementsCount = paths.size();
+
+ Y_VERIFY(elementsCount);
+
+ auto schemeCacheRequest = MakeHolder<TSchemeCacheNavigate>();
+
+ schemeCacheRequest->ResultSet.resize(elementsCount);
+
+ for (size_t i = 0; i < elementsCount; ++i) {
+ schemeCacheRequest->ResultSet[i].Path = paths[i];
+
+ schemeCacheRequest->ResultSet[i].Operation =
+ ((CurrentDepth == MaxDepth) ? TSchemeCacheNavigate::OpPath : TSchemeCacheNavigate::OpList);
+ }
+
+ Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release()));
+ }
+
+ STATEFN(Working) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse);
+ }
+ }
+
+ void OnFinishedTraversal(bool success) {
+ auto ev = MakeHolder<TSqsEvents::TEvSchemeTraversalResult>(success);
+
+ if (success) {
+ ev->RootHolder = std::move(RootHolder);
+ }
+
+ Send(ParentId, ev.Release());
+
+ PassAway();
+ }
+
+ void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get();
+
+ if (navigate->ErrorCount) {
+ OnFinishedTraversal(false);
+ return;
+ }
+
+ TVector<TSchemePath> newPathsToTraverse;
+
+ ++CurrentDepth;
+
+ for (const auto& entry : navigate->ResultSet) {
+ auto& node = FindOrAllocateSchemeNode(&*RootHolder, entry.Path);
+
+ node.Path = entry.Path;
+ node.Kind = entry.Kind;
+ node.CreationTs = TInstant::MilliSeconds(entry.CreateStep);
+
+ if (entry.ListNodeEntry && CurrentDepth <= MaxDepth) {
+ for (const auto& child : entry.ListNodeEntry->Children) {
+ auto yetAnotherPathToTraverse = entry.Path;
yetAnotherPathToTraverse.push_back(child.Name);
-
- newPathsToTraverse.push_back(std::move(yetAnotherPathToTraverse));
- }
- }
- }
-
- if (newPathsToTraverse.empty()) {
- OnFinishedTraversal(true);
- } else {
- SendListChildrenRequest(newPathsToTraverse);
- }
- }
-
-private:
+
+ newPathsToTraverse.push_back(std::move(yetAnotherPathToTraverse));
+ }
+ }
+ }
+
+ if (newPathsToTraverse.empty()) {
+ OnFinishedTraversal(true);
+ } else {
+ SendListChildrenRequest(newPathsToTraverse);
+ }
+ }
+
+private:
const TActorId ParentId;
const TActorId SchemeCacheId;
- const TVector<TSchemePath> PathsToTraverse;
-
- const ui64 MaxDepth;
- ui64 CurrentDepth;
-
- THolder<TSchemeNode> RootHolder;
-};
-
-class TGarbageSearcher : public TActorBootstrapped<TGarbageSearcher> {
-public:
+ const TVector<TSchemePath> PathsToTraverse;
+
+ const ui64 MaxDepth;
+ ui64 CurrentDepth;
+
+ THolder<TSchemeNode> RootHolder;
+};
+
+class TGarbageSearcher : public TActorBootstrapped<TGarbageSearcher> {
+public:
TGarbageSearcher(const TActorId parentId,
const TActorId schemeCacheId,
const TActorId queuesListReaderId,
- const ui64 minimumItemAgeSeconds)
- : ParentId(parentId)
- , SchemeCacheId(schemeCacheId)
- , QueuesListReaderId(queuesListReaderId)
- , MinimumItemAge(TDuration::Seconds(minimumItemAgeSeconds))
- {
- Y_VERIFY(parentId);
- Y_VERIFY(schemeCacheId);
- Y_VERIFY(queuesListReaderId);
- }
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
- }
-
- void Bootstrap() {
- Become(&TThis::ListingAccounts);
-
- RootPath = SplitPath(Cfg().GetRoot());
-
- // list accounts and system tables only
- Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {RootPath}, 1));
-
- Send(QueuesListReaderId, new TSqsEvents::TEvReadQueuesList());
-
- RequiredResponses = 2;
- }
-
-private:
- void ProcessQueuesListingResult(TSqsEvents::TEvQueuesList::TPtr& ev) {
- if (ev->Get()->Success) {
- SortedQueues = std::move(ev->Get()->SortedQueues);
-
- CompleteListing();
- } else {
- ReplyToParentAndDie(false);
- }
- }
-
- void ProcessAccountsListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) {
- if (ev->Get()->Success) {
- auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, RootPath);
-
- Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath);
-
- for (const auto& [name, child] : node.Children) {
- if (child.Kind == TSchemeCacheNavigate::EKind::KindPath && !name.StartsWith(".")) {
- AccountsToProcess.push(name);
- }
- }
-
- CompleteListing();
- } else {
- ReplyToParentAndDie(false);
- }
- }
-
- void CompleteListing() {
- Y_VERIFY(RequiredResponses);
-
- if (!--RequiredResponses) {
- Become(&TThis::ProcessingAccounts);
-
- ListNextAccountOrReply();
-
- return;
- }
- }
-
- void ListNextAccountOrReply() {
- if (AccountsToProcess.size()) {
- CurrentAccount = AccountsToProcess.front();
- AccountsToProcess.pop();
-
- CurrentAccountPath = RootPath;
- CurrentAccountPath.push_back(CurrentAccount);
-
- // depth == 2 since the path pattern is account/queue/queue_version
- Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {CurrentAccountPath}, 2));
- } else {
- ReplyToParentAndDie(true);
- }
- }
-
- STATEFN(ListingAccounts) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessAccountsListingResult);
- hFunc(TSqsEvents::TEvQueuesList, ProcessQueuesListingResult);
- }
- }
-
- // per-account processing section
-
- STATEFN(ProcessingAccounts) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessSingleAccount);
- }
- }
-
- void ProcessSingleAccount(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) {
- if (ev->Get()->Success) {
- auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, CurrentAccountPath);
-
- // an account is always a directory
- Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath);
-
- for (auto& [queueName, queueNode] : node.Children) {
- // ignore tables (e. g. local "Queues" table)
- if (queueNode.Kind == TSchemeCacheNavigate::EKind::KindPath) {
- ProcessSingleQueue(queueName, queueNode);
- }
- }
- } else {
- // Skip account on an error. Possible reason: some elements were removed during a listing
- }
-
- ListNextAccountOrReply();
- }
-
- void ProcessSingleQueue(const TString& queueName, TSchemeNode& node) {
- TQueueRecord fakeRecord;
- fakeRecord.UserName = CurrentAccount;
- fakeRecord.QueueName = queueName;
-
- auto it = std::lower_bound(SortedQueues.begin(), SortedQueues.end(), fakeRecord);
- if (it != SortedQueues.end() && it->UserName == CurrentAccount && it->QueueName == queueName) {
- const bool isLegacyQueue = it->Version == 0;
-
- if (isLegacyQueue) {
- // TODO: check State field and report
- } else {
- const TString versionStr = "v" + ToString(it->Version);
- for (auto& [name, child] : node.Children) {
- // Do not ever touch fresh queues or actual versions
- const TDuration itemAge = TActivationContext::AsActorContext().Now() - node.CreationTs;
- if (name != versionStr && itemAge >= MinimumItemAge) {
- AddGarbageHint(CurrentAccount, std::move(child), TGarbageHint::EReason::UnusedVersion);
- }
- }
- }
- } else {
- AddGarbageHint(CurrentAccount, std::move(node), TGarbageHint::EReason::UnregisteredQueue);
- }
- }
-
- void AddGarbageHint(const TString& account, TSchemeNode&& node, const TGarbageHint::EReason reason) {
- // remove excessive data
- node.Children.clear();
-
- TGarbageHint hint{account, std::move(node), reason};
-
- const auto path = CanonizePath(hint.SchemeNode.Path);
-
- GarbageHints[path] = std::move(hint);
- }
-
- void ReplyToParentAndDie(bool success) {
- auto ev = MakeHolder<TSqsEvents::TEvGarbageSearchResult>(success);
-
- if (success) {
- ev->GarbageHints = std::move(GarbageHints);
- }
-
- Send(ParentId, ev.Release());
-
- PassAway();
- }
-
-private:
+ const ui64 minimumItemAgeSeconds)
+ : ParentId(parentId)
+ , SchemeCacheId(schemeCacheId)
+ , QueuesListReaderId(queuesListReaderId)
+ , MinimumItemAge(TDuration::Seconds(minimumItemAgeSeconds))
+ {
+ Y_VERIFY(parentId);
+ Y_VERIFY(schemeCacheId);
+ Y_VERIFY(queuesListReaderId);
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
+ }
+
+ void Bootstrap() {
+ Become(&TThis::ListingAccounts);
+
+ RootPath = SplitPath(Cfg().GetRoot());
+
+ // list accounts and system tables only
+ Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {RootPath}, 1));
+
+ Send(QueuesListReaderId, new TSqsEvents::TEvReadQueuesList());
+
+ RequiredResponses = 2;
+ }
+
+private:
+ void ProcessQueuesListingResult(TSqsEvents::TEvQueuesList::TPtr& ev) {
+ if (ev->Get()->Success) {
+ SortedQueues = std::move(ev->Get()->SortedQueues);
+
+ CompleteListing();
+ } else {
+ ReplyToParentAndDie(false);
+ }
+ }
+
+ void ProcessAccountsListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) {
+ if (ev->Get()->Success) {
+ auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, RootPath);
+
+ Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath);
+
+ for (const auto& [name, child] : node.Children) {
+ if (child.Kind == TSchemeCacheNavigate::EKind::KindPath && !name.StartsWith(".")) {
+ AccountsToProcess.push(name);
+ }
+ }
+
+ CompleteListing();
+ } else {
+ ReplyToParentAndDie(false);
+ }
+ }
+
+ void CompleteListing() {
+ Y_VERIFY(RequiredResponses);
+
+ if (!--RequiredResponses) {
+ Become(&TThis::ProcessingAccounts);
+
+ ListNextAccountOrReply();
+
+ return;
+ }
+ }
+
+ void ListNextAccountOrReply() {
+ if (AccountsToProcess.size()) {
+ CurrentAccount = AccountsToProcess.front();
+ AccountsToProcess.pop();
+
+ CurrentAccountPath = RootPath;
+ CurrentAccountPath.push_back(CurrentAccount);
+
+ // depth == 2 since the path pattern is account/queue/queue_version
+ Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {CurrentAccountPath}, 2));
+ } else {
+ ReplyToParentAndDie(true);
+ }
+ }
+
+ STATEFN(ListingAccounts) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessAccountsListingResult);
+ hFunc(TSqsEvents::TEvQueuesList, ProcessQueuesListingResult);
+ }
+ }
+
+ // per-account processing section
+
+ STATEFN(ProcessingAccounts) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessSingleAccount);
+ }
+ }
+
+ void ProcessSingleAccount(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) {
+ if (ev->Get()->Success) {
+ auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, CurrentAccountPath);
+
+ // an account is always a directory
+ Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath);
+
+ for (auto& [queueName, queueNode] : node.Children) {
+ // ignore tables (e. g. local "Queues" table)
+ if (queueNode.Kind == TSchemeCacheNavigate::EKind::KindPath) {
+ ProcessSingleQueue(queueName, queueNode);
+ }
+ }
+ } else {
+ // Skip account on an error. Possible reason: some elements were removed during a listing
+ }
+
+ ListNextAccountOrReply();
+ }
+
+ void ProcessSingleQueue(const TString& queueName, TSchemeNode& node) {
+ TQueueRecord fakeRecord;
+ fakeRecord.UserName = CurrentAccount;
+ fakeRecord.QueueName = queueName;
+
+ auto it = std::lower_bound(SortedQueues.begin(), SortedQueues.end(), fakeRecord);
+ if (it != SortedQueues.end() && it->UserName == CurrentAccount && it->QueueName == queueName) {
+ const bool isLegacyQueue = it->Version == 0;
+
+ if (isLegacyQueue) {
+ // TODO: check State field and report
+ } else {
+ const TString versionStr = "v" + ToString(it->Version);
+ for (auto& [name, child] : node.Children) {
+ // Do not ever touch fresh queues or actual versions
+ const TDuration itemAge = TActivationContext::AsActorContext().Now() - node.CreationTs;
+ if (name != versionStr && itemAge >= MinimumItemAge) {
+ AddGarbageHint(CurrentAccount, std::move(child), TGarbageHint::EReason::UnusedVersion);
+ }
+ }
+ }
+ } else {
+ AddGarbageHint(CurrentAccount, std::move(node), TGarbageHint::EReason::UnregisteredQueue);
+ }
+ }
+
+ void AddGarbageHint(const TString& account, TSchemeNode&& node, const TGarbageHint::EReason reason) {
+ // remove excessive data
+ node.Children.clear();
+
+ TGarbageHint hint{account, std::move(node), reason};
+
+ const auto path = CanonizePath(hint.SchemeNode.Path);
+
+ GarbageHints[path] = std::move(hint);
+ }
+
+ void ReplyToParentAndDie(bool success) {
+ auto ev = MakeHolder<TSqsEvents::TEvGarbageSearchResult>(success);
+
+ if (success) {
+ ev->GarbageHints = std::move(GarbageHints);
+ }
+
+ Send(ParentId, ev.Release());
+
+ PassAway();
+ }
+
+private:
const TActorId ParentId;
const TActorId SchemeCacheId;
const TActorId QueuesListReaderId;
- const TDuration MinimumItemAge;
-
- TSchemePath RootPath;
-
- std::queue<TString> AccountsToProcess;
- std::vector<TQueueRecord> SortedQueues;
-
- ui64 RequiredResponses = 0;
-
- TString CurrentAccount;
- TSchemePath CurrentAccountPath;
-
- THashMap<TString, TGarbageHint> GarbageHints;
-};
-
-class TGarbageCleaner : public TActorBootstrapped<TGarbageCleaner> {
-public:
+ const TDuration MinimumItemAge;
+
+ TSchemePath RootPath;
+
+ std::queue<TString> AccountsToProcess;
+ std::vector<TQueueRecord> SortedQueues;
+
+ ui64 RequiredResponses = 0;
+
+ TString CurrentAccount;
+ TSchemePath CurrentAccountPath;
+
+ THashMap<TString, TGarbageHint> GarbageHints;
+};
+
+class TGarbageCleaner : public TActorBootstrapped<TGarbageCleaner> {
+public:
TGarbageCleaner(const TActorId parentId,
const TActorId schemeCacheId,
- TGarbageHint&& garbageHint)
- : ParentId(parentId)
- , SchemeCacheId(schemeCacheId)
- , GarbageHint(std::move(garbageHint))
- {
- Y_VERIFY(parentId);
- Y_VERIFY(schemeCacheId);
- Y_VERIFY(GarbageHint.Reason != TGarbageHint::EReason::Unknown);
- Y_VERIFY(GarbageHint.SchemeNode.Path.size());
- }
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
- }
-
- void Bootstrap() {
- Become(&TThis::Working);
-
- StartedAt = TActivationContext::AsActorContext().Now();
-
- Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {GarbageHint.SchemeNode.Path}));
- }
-
-private:
- STATEFN(Working) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessGarbageListingResult);
- hFunc(TSqsEvents::TEvExecuted, OnNodeRemoved);
- }
- }
-
- void DFS(const TSchemeNode& node) {
- for (const auto& [name, child] : node.Children) {
- DFS(child);
- }
-
- NodesToRemove.push(node);
- }
-
- void ProcessGarbageListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) {
- if (ev->Get()->Success) {
- auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, GarbageHint.SchemeNode.Path);
-
- Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath);
-
- DFS(node);
-
- RemoveNextNodeOrDie();
- } else {
- LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to list nodes at " << CanonizePath(GarbageHint.SchemeNode.Path));
-
- ReportToParentAndDie(false);
- }
- }
-
- void RemoveNextNodeOrDie() {
- if (!NodesToRemove.empty()) {
- CurrentNode = std::move(NodesToRemove.front());
- NodesToRemove.pop();
-
- RemoveCurrentNode();
- } else {
- LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " finished cleaning at " << CanonizePath(GarbageHint.SchemeNode.Path));
-
- ReportToParentAndDie(true);
- }
- }
-
- void ReportToParentAndDie(const bool success) {
- auto ev = MakeHolder<TSqsEvents::TEvGarbageCleaningResult>(success);
-
- ev->Record.Account = GarbageHint.Account;
- ev->Record.HintPath = CanonizePath(GarbageHint.SchemeNode.Path);
- ev->Record.RemovedNodes = std::move(RemovedNodes);
- ev->Record.StartedAt = StartedAt;
- ev->Record.Duration = TActivationContext::AsActorContext().Now() - StartedAt;
-
- Send(ParentId, ev.Release());
-
- PassAway();
- }
-
- TQueuePath MakeAndVerifyQueuePath(const TSchemePath& path) const {
- // A certain dir structure is assumed here: SqsRoot/Account/Queue/Version
- // All path components except "Version" are required
- // The code guarantees scheme modifications locality
-
- Y_VERIFY(CanonizePath(path).StartsWith(CanonizePath(GarbageHint.SchemeNode.Path)));
-
- auto rootPath = SplitPath(Cfg().GetRoot());
-
- auto rootPathIt = rootPath.begin();
- auto pathIt = path.begin();
-
- while (rootPathIt != rootPath.end()) {
- Y_VERIFY(*rootPathIt++ == *pathIt++);
- }
-
- TQueuePath result;
- result.Root = Cfg().GetRoot();
-
- Y_VERIFY(pathIt != path.end());
-
- result.UserName = *pathIt++;
-
- Y_VERIFY(result.UserName == GarbageHint.Account);
-
- Y_VERIFY(pathIt != path.end());
-
- result.QueueName = *pathIt++;
-
- if (pathIt != path.end()) {
- result.VersionSuffix = *pathIt;
- }
-
- return result;
- }
-
- void RemoveCurrentNode() const {
- auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
-
- auto path = CurrentNode.Path;
-
- Y_VERIFY(path.size() > 1);
-
- const auto name = path.back();
-
- path.pop_back();
-
- trans->SetWorkingDir(CanonizePath(path));
-
- trans->MutableDrop()->SetName(name);
-
- if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindPath) {
+ TGarbageHint&& garbageHint)
+ : ParentId(parentId)
+ , SchemeCacheId(schemeCacheId)
+ , GarbageHint(std::move(garbageHint))
+ {
+ Y_VERIFY(parentId);
+ Y_VERIFY(schemeCacheId);
+ Y_VERIFY(GarbageHint.Reason != TGarbageHint::EReason::Unknown);
+ Y_VERIFY(GarbageHint.SchemeNode.Path.size());
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
+ }
+
+ void Bootstrap() {
+ Become(&TThis::Working);
+
+ StartedAt = TActivationContext::AsActorContext().Now();
+
+ Register(new TSchemeTraversalActor(SelfId(), SchemeCacheId, {GarbageHint.SchemeNode.Path}));
+ }
+
+private:
+ STATEFN(Working) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TSqsEvents::TEvSchemeTraversalResult, ProcessGarbageListingResult);
+ hFunc(TSqsEvents::TEvExecuted, OnNodeRemoved);
+ }
+ }
+
+ void DFS(const TSchemeNode& node) {
+ for (const auto& [name, child] : node.Children) {
+ DFS(child);
+ }
+
+ NodesToRemove.push(node);
+ }
+
+ void ProcessGarbageListingResult(TSqsEvents::TEvSchemeTraversalResult::TPtr& ev) {
+ if (ev->Get()->Success) {
+ auto& node = FindOrAllocateSchemeNode(&*ev->Get()->RootHolder, GarbageHint.SchemeNode.Path);
+
+ Y_VERIFY(node.Kind = TSchemeCacheNavigate::EKind::KindPath);
+
+ DFS(node);
+
+ RemoveNextNodeOrDie();
+ } else {
+ LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to list nodes at " << CanonizePath(GarbageHint.SchemeNode.Path));
+
+ ReportToParentAndDie(false);
+ }
+ }
+
+ void RemoveNextNodeOrDie() {
+ if (!NodesToRemove.empty()) {
+ CurrentNode = std::move(NodesToRemove.front());
+ NodesToRemove.pop();
+
+ RemoveCurrentNode();
+ } else {
+ LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " finished cleaning at " << CanonizePath(GarbageHint.SchemeNode.Path));
+
+ ReportToParentAndDie(true);
+ }
+ }
+
+ void ReportToParentAndDie(const bool success) {
+ auto ev = MakeHolder<TSqsEvents::TEvGarbageCleaningResult>(success);
+
+ ev->Record.Account = GarbageHint.Account;
+ ev->Record.HintPath = CanonizePath(GarbageHint.SchemeNode.Path);
+ ev->Record.RemovedNodes = std::move(RemovedNodes);
+ ev->Record.StartedAt = StartedAt;
+ ev->Record.Duration = TActivationContext::AsActorContext().Now() - StartedAt;
+
+ Send(ParentId, ev.Release());
+
+ PassAway();
+ }
+
+ TQueuePath MakeAndVerifyQueuePath(const TSchemePath& path) const {
+ // A certain dir structure is assumed here: SqsRoot/Account/Queue/Version
+ // All path components except "Version" are required
+ // The code guarantees scheme modifications locality
+
+ Y_VERIFY(CanonizePath(path).StartsWith(CanonizePath(GarbageHint.SchemeNode.Path)));
+
+ auto rootPath = SplitPath(Cfg().GetRoot());
+
+ auto rootPathIt = rootPath.begin();
+ auto pathIt = path.begin();
+
+ while (rootPathIt != rootPath.end()) {
+ Y_VERIFY(*rootPathIt++ == *pathIt++);
+ }
+
+ TQueuePath result;
+ result.Root = Cfg().GetRoot();
+
+ Y_VERIFY(pathIt != path.end());
+
+ result.UserName = *pathIt++;
+
+ Y_VERIFY(result.UserName == GarbageHint.Account);
+
+ Y_VERIFY(pathIt != path.end());
+
+ result.QueueName = *pathIt++;
+
+ if (pathIt != path.end()) {
+ result.VersionSuffix = *pathIt;
+ }
+
+ return result;
+ }
+
+ void RemoveCurrentNode() const {
+ auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
+
+ auto path = CurrentNode.Path;
+
+ Y_VERIFY(path.size() > 1);
+
+ const auto name = path.back();
+
+ path.pop_back();
+
+ trans->SetWorkingDir(CanonizePath(path));
+
+ trans->MutableDrop()->SetName(name);
+
+ if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindPath) {
trans->SetOperationType(NKikimrSchemeOp::ESchemeOpRmDir);
- } else if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindTable) {
+ } else if (CurrentNode.Kind == TSchemeCacheNavigate::EKind::KindTable) {
trans->SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable);
- } else {
- Y_VERIFY("Unexpected node kind");
- }
-
- LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " attempts to remove the node " << CanonizePath(CurrentNode.Path));
-
- const TQueuePath queuePath = MakeAndVerifyQueuePath(CurrentNode.Path);
-
- Register(new TMiniKqlExecutionActor(SelfId(), GARBAGE_CLEANER_LABEL + "Tx", std::move(ev), false, queuePath, nullptr));
- }
-
- void OnNodeRemoved(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const auto status = record.GetStatus();
-
- if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
- const auto path = CanonizePath(CurrentNode.Path);
-
- RemovedNodes.push_back(path);
-
- LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " removed the node " << path);
-
- RemoveNextNodeOrDie();
- } else {
- LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to remove the node " << CanonizePath(CurrentNode.Path));
-
- ReportToParentAndDie(false);
- }
- }
-
-private:
+ } else {
+ Y_VERIFY("Unexpected node kind");
+ }
+
+ LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " attempts to remove the node " << CanonizePath(CurrentNode.Path));
+
+ const TQueuePath queuePath = MakeAndVerifyQueuePath(CurrentNode.Path);
+
+ Register(new TMiniKqlExecutionActor(SelfId(), GARBAGE_CLEANER_LABEL + "Tx", std::move(ev), false, queuePath, nullptr));
+ }
+
+ void OnNodeRemoved(TSqsEvents::TEvExecuted::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+ const auto status = record.GetStatus();
+
+ if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
+ const auto path = CanonizePath(CurrentNode.Path);
+
+ RemovedNodes.push_back(path);
+
+ LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " removed the node " << path);
+
+ RemoveNextNodeOrDie();
+ } else {
+ LOG_SQS_INFO(GARBAGE_CLEANER_LABEL << " failed to remove the node " << CanonizePath(CurrentNode.Path));
+
+ ReportToParentAndDie(false);
+ }
+ }
+
+private:
const TActorId ParentId;
const TActorId SchemeCacheId;
- const TGarbageHint GarbageHint;
- std::queue<TSchemeNode> NodesToRemove;
- TSchemeNode CurrentNode;
- TVector<TString> RemovedNodes;
- TInstant StartedAt;
-};
-
-class TGarbageCollector : public TActorBootstrapped<TGarbageCollector> {
-public:
+ const TGarbageHint GarbageHint;
+ std::queue<TSchemeNode> NodesToRemove;
+ TSchemeNode CurrentNode;
+ TVector<TString> RemovedNodes;
+ TInstant StartedAt;
+};
+
+class TGarbageCollector : public TActorBootstrapped<TGarbageCollector> {
+public:
TGarbageCollector(const TActorId schemeCacheId, const TActorId queuesListReaderId)
- : SchemeCacheId(schemeCacheId)
- , QueuesListReaderId(queuesListReaderId)
- {
- Y_VERIFY(schemeCacheId);
- Y_VERIFY(queuesListReaderId);
- Y_UNUSED(PrintAllPaths);
- }
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
- }
-
- void Bootstrap() {
- Become(&TThis::Working);
-
- RegisterMonitoringPage();
- }
-
- void RegisterMonitoringPage() const {
- NActors::TMon* mon = AppData()->Mon;
- if (mon) {
- NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors");
- mon->RegisterActorPage(page, "sqsgc", "SQS Garbage Collector", false,
- TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
- }
- }
-
- void ProcessGarbageHints(TSqsEvents::TEvGarbageSearchResult::TPtr& ev) {
- Scanning = false;
-
- if (ev->Get()->Success) {
- CurrentGarbageHints = std::move(ev->Get()->GarbageHints);
- }
- }
-
- void InitRescanAndReport(NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream str;
-
- HTML(str) {
- if (Scanning) {
- str << "Already scanning. Patience is a virtue, dear user";
- } else {
- const TCgiParameters& params = ev->Get()->Request.GetParams();
- if (params.Has(MINIMUM_ITEM_AGE_SECONDS)) {
- const TString& minimumItemAgeSecondsParam = params.Get(MINIMUM_ITEM_AGE_SECONDS);
-
- ui64 minimumItemAgeSeconds = 0;
- if (TryFromString<ui64>(minimumItemAgeSecondsParam, minimumItemAgeSeconds)) {
- Scanning = true;
-
- Register(new TGarbageSearcher(SelfId(),
- SchemeCacheId,
- QueuesListReaderId,
- minimumItemAgeSeconds));
-
- str << "Started new scan process";
- } else {
- str << "Can't parse minimum item age param value: " << minimumItemAgeSecondsParam;
- }
- } else {
- str << "Minimum item age param is missing";
- }
- }
- str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>";
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
- }
-
- void StartCleaner(TGarbageHint&& garbageHint) {
- Register(new TGarbageCleaner(SelfId(), SchemeCacheId, std::move(garbageHint)));
- }
-
- void InitCleaningAndReport(NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream str;
-
- HTML(str) {
- const TCgiParameters& params = ev->Get()->Request.GetParams();
-
- if (params.Has(PATH_TO_CLEAN)) {
- const TString& pathToClean = params.Get(PATH_TO_CLEAN);
- auto it = CurrentGarbageHints.find(pathToClean);
- if (it == CurrentGarbageHints.end()) {
- str << "Specified path doesn't match any garbage hint: " << params.Get(PATH_TO_CLEAN);
- } else {
- StartCleaner(std::move(it->second));
-
- CurrentGarbageHints.erase(it);
-
- str << "Started cleaning process for " << params.Get(PATH_TO_CLEAN);
- }
- } else {
- str << "Path to clean is missing";
- }
- str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>";
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
- }
-
- void InitClearHistoryAndReport(NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream str;
-
- CleaningHistory.clear();
-
- HTML(str) {
- str << "Cleaning history was reset";
- str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>";
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
- }
-
- static void AddCleanGarbageLink(TStringStream& str, const TString& pathToClean) {
- TCgiParameters params;
- params.InsertUnescaped(PATH_TO_CLEAN, pathToClean);
-
- str << "<a href=\"/actors/sqsgc/clean?" << params() << "\">" << "Clean" << "</a>";
- }
-
- static void AddRescanForm(TStringStream& str) {
- str << "<form method=\"GET\" action=\"/actors/sqsgc/rescan\" id=\"tblMonSQSGCfrm\" name=\"tblMonSQSGCfrm\">" << Endl;
- str << "<legend> Rescan structure and filter by item age: </legend>";
- str << "<label> " << "Minimum item age (seconds): " << " <input type=\"number\" id=\"" << "lblRescan";
- str << "\" name=\"" << MINIMUM_ITEM_AGE_SECONDS << "\"" << " value=\"" << Cfg().GetMinimumGarbageAgeSeconds() << "\"";
- str << " min=\"0\" /> </label>";
- str << "<input class=\"btn btn-default\" type=\"submit\" value=\"Rescan\"/>" << Endl;
- str << "</form>" << Endl;
- }
-
- void RenderGarbageHints(TStringStream& str) const {
- if (CurrentGarbageHints.empty()) {
- return;
- }
-
- IOutputStream& __stream(str);
-
- H4() {
- str << "Garbage hints";
- }
- TABLE_SORTABLE_CLASS("hints-table") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { str << "Path"; }
- TABLEH() { str << "Created at (UTC)"; }
- TABLEH() { str << "Reason"; }
- TABLEH() { str << "Action"; }
- }
- }
- TABLEBODY() {
- for (const auto& [pathToClean, hint] : CurrentGarbageHints) {
- TABLER() {
- TABLED() { str << pathToClean; }
- TABLED() { str << hint.SchemeNode.CreationTs.ToRfc822String(); }
- TABLED() { str << hint.Reason; }
- TABLED() { AddCleanGarbageLink(str, pathToClean); }
- }
- }
- }
- }
- }
-
- void RenderCleaningResults(TStringStream& str) const {
- if (CleaningHistory.empty()) {
- return;
- }
-
- IOutputStream& __stream(str);
-
- H4() {
- str << "Cleaning history";
- }
- TABLE_SORTABLE_CLASS("cr-table") {
- TABLEHEAD() {
- TABLER() {
- TABLEH() { str << "Account"; }
- TABLEH() { str << "Hint path"; }
- TABLEH() { str << "Started at (UTC)"; }
- TABLEH() { str << "Duration (ms)"; }
- TABLEH() { str << "Nodes removed"; }
- TABLEH() { str << "Result"; }
- }
- }
- TABLEBODY() {
- for (const auto& result : CleaningHistory) {
- TABLER() {
- TABLED() { str << result.Account; }
- TABLED() { str << result.HintPath; }
- TABLED() { str << result.StartedAt.ToRfc822String(); }
- TABLED() { str << result.Duration.MilliSeconds(); }
- TABLED() { str << result.RemovedNodes.size(); }
- TABLED() { str << (result.Success ? "OK" : "FAIL"); }
- }
- }
- }
- }
- str << "<a href=\"/actors/sqsgc/clear_history\">" << "Reset local cleaning history" << "</a>";
- }
-
- void RenderMainControlPage(NMon::TEvHttpInfo::TPtr& ev) {
- TStringStream str;
-
- HTML(str) {
- AddRescanForm(str);
- RenderGarbageHints(str);
- RenderCleaningResults(str);
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
- }
-
- void HandleCleaningResult(TSqsEvents::TEvGarbageCleaningResult::TPtr& ev) {
- CleaningHistory.push_back(std::move(ev->Get()->Record));
- }
-
- void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) {
- const TStringBuf path = ev->Get()->Request.GetPath();
-
- if (path.EndsWith("/rescan")) {
- InitRescanAndReport(ev);
- } else if (path.EndsWith("/clean")) {
- InitCleaningAndReport(ev);
- } else if (path.EndsWith("/clear_history")) {
- InitClearHistoryAndReport(ev);
- } else {
- RenderMainControlPage(ev);
- }
- }
-
- STATEFN(Working) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TSqsEvents::TEvGarbageSearchResult, ProcessGarbageHints);
- hFunc(TSqsEvents::TEvGarbageCleaningResult, HandleCleaningResult);
- hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
- }
- }
-
-private:
+ : SchemeCacheId(schemeCacheId)
+ , QueuesListReaderId(queuesListReaderId)
+ {
+ Y_VERIFY(schemeCacheId);
+ Y_VERIFY(queuesListReaderId);
+ Y_UNUSED(PrintAllPaths);
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::SQS_GARBAGE_COLLECTOR_ACTOR;
+ }
+
+ void Bootstrap() {
+ Become(&TThis::Working);
+
+ RegisterMonitoringPage();
+ }
+
+ void RegisterMonitoringPage() const {
+ NActors::TMon* mon = AppData()->Mon;
+ if (mon) {
+ NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors");
+ mon->RegisterActorPage(page, "sqsgc", "SQS Garbage Collector", false,
+ TlsActivationContext->ExecutorThread.ActorSystem, SelfId());
+ }
+ }
+
+ void ProcessGarbageHints(TSqsEvents::TEvGarbageSearchResult::TPtr& ev) {
+ Scanning = false;
+
+ if (ev->Get()->Success) {
+ CurrentGarbageHints = std::move(ev->Get()->GarbageHints);
+ }
+ }
+
+ void InitRescanAndReport(NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream str;
+
+ HTML(str) {
+ if (Scanning) {
+ str << "Already scanning. Patience is a virtue, dear user";
+ } else {
+ const TCgiParameters& params = ev->Get()->Request.GetParams();
+ if (params.Has(MINIMUM_ITEM_AGE_SECONDS)) {
+ const TString& minimumItemAgeSecondsParam = params.Get(MINIMUM_ITEM_AGE_SECONDS);
+
+ ui64 minimumItemAgeSeconds = 0;
+ if (TryFromString<ui64>(minimumItemAgeSecondsParam, minimumItemAgeSeconds)) {
+ Scanning = true;
+
+ Register(new TGarbageSearcher(SelfId(),
+ SchemeCacheId,
+ QueuesListReaderId,
+ minimumItemAgeSeconds));
+
+ str << "Started new scan process";
+ } else {
+ str << "Can't parse minimum item age param value: " << minimumItemAgeSecondsParam;
+ }
+ } else {
+ str << "Minimum item age param is missing";
+ }
+ }
+ str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>";
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+ void StartCleaner(TGarbageHint&& garbageHint) {
+ Register(new TGarbageCleaner(SelfId(), SchemeCacheId, std::move(garbageHint)));
+ }
+
+ void InitCleaningAndReport(NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream str;
+
+ HTML(str) {
+ const TCgiParameters& params = ev->Get()->Request.GetParams();
+
+ if (params.Has(PATH_TO_CLEAN)) {
+ const TString& pathToClean = params.Get(PATH_TO_CLEAN);
+ auto it = CurrentGarbageHints.find(pathToClean);
+ if (it == CurrentGarbageHints.end()) {
+ str << "Specified path doesn't match any garbage hint: " << params.Get(PATH_TO_CLEAN);
+ } else {
+ StartCleaner(std::move(it->second));
+
+ CurrentGarbageHints.erase(it);
+
+ str << "Started cleaning process for " << params.Get(PATH_TO_CLEAN);
+ }
+ } else {
+ str << "Path to clean is missing";
+ }
+ str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>";
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+ void InitClearHistoryAndReport(NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream str;
+
+ CleaningHistory.clear();
+
+ HTML(str) {
+ str << "Cleaning history was reset";
+ str << "<br></br>" << "<a href=\"/actors/sqsgc\">Back</a>";
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+ static void AddCleanGarbageLink(TStringStream& str, const TString& pathToClean) {
+ TCgiParameters params;
+ params.InsertUnescaped(PATH_TO_CLEAN, pathToClean);
+
+ str << "<a href=\"/actors/sqsgc/clean?" << params() << "\">" << "Clean" << "</a>";
+ }
+
+ static void AddRescanForm(TStringStream& str) {
+ str << "<form method=\"GET\" action=\"/actors/sqsgc/rescan\" id=\"tblMonSQSGCfrm\" name=\"tblMonSQSGCfrm\">" << Endl;
+ str << "<legend> Rescan structure and filter by item age: </legend>";
+ str << "<label> " << "Minimum item age (seconds): " << " <input type=\"number\" id=\"" << "lblRescan";
+ str << "\" name=\"" << MINIMUM_ITEM_AGE_SECONDS << "\"" << " value=\"" << Cfg().GetMinimumGarbageAgeSeconds() << "\"";
+ str << " min=\"0\" /> </label>";
+ str << "<input class=\"btn btn-default\" type=\"submit\" value=\"Rescan\"/>" << Endl;
+ str << "</form>" << Endl;
+ }
+
+ void RenderGarbageHints(TStringStream& str) const {
+ if (CurrentGarbageHints.empty()) {
+ return;
+ }
+
+ IOutputStream& __stream(str);
+
+ H4() {
+ str << "Garbage hints";
+ }
+ TABLE_SORTABLE_CLASS("hints-table") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { str << "Path"; }
+ TABLEH() { str << "Created at (UTC)"; }
+ TABLEH() { str << "Reason"; }
+ TABLEH() { str << "Action"; }
+ }
+ }
+ TABLEBODY() {
+ for (const auto& [pathToClean, hint] : CurrentGarbageHints) {
+ TABLER() {
+ TABLED() { str << pathToClean; }
+ TABLED() { str << hint.SchemeNode.CreationTs.ToRfc822String(); }
+ TABLED() { str << hint.Reason; }
+ TABLED() { AddCleanGarbageLink(str, pathToClean); }
+ }
+ }
+ }
+ }
+ }
+
+ void RenderCleaningResults(TStringStream& str) const {
+ if (CleaningHistory.empty()) {
+ return;
+ }
+
+ IOutputStream& __stream(str);
+
+ H4() {
+ str << "Cleaning history";
+ }
+ TABLE_SORTABLE_CLASS("cr-table") {
+ TABLEHEAD() {
+ TABLER() {
+ TABLEH() { str << "Account"; }
+ TABLEH() { str << "Hint path"; }
+ TABLEH() { str << "Started at (UTC)"; }
+ TABLEH() { str << "Duration (ms)"; }
+ TABLEH() { str << "Nodes removed"; }
+ TABLEH() { str << "Result"; }
+ }
+ }
+ TABLEBODY() {
+ for (const auto& result : CleaningHistory) {
+ TABLER() {
+ TABLED() { str << result.Account; }
+ TABLED() { str << result.HintPath; }
+ TABLED() { str << result.StartedAt.ToRfc822String(); }
+ TABLED() { str << result.Duration.MilliSeconds(); }
+ TABLED() { str << result.RemovedNodes.size(); }
+ TABLED() { str << (result.Success ? "OK" : "FAIL"); }
+ }
+ }
+ }
+ }
+ str << "<a href=\"/actors/sqsgc/clear_history\">" << "Reset local cleaning history" << "</a>";
+ }
+
+ void RenderMainControlPage(NMon::TEvHttpInfo::TPtr& ev) {
+ TStringStream str;
+
+ HTML(str) {
+ AddRescanForm(str);
+ RenderGarbageHints(str);
+ RenderCleaningResults(str);
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str()));
+ }
+
+ void HandleCleaningResult(TSqsEvents::TEvGarbageCleaningResult::TPtr& ev) {
+ CleaningHistory.push_back(std::move(ev->Get()->Record));
+ }
+
+ void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) {
+ const TStringBuf path = ev->Get()->Request.GetPath();
+
+ if (path.EndsWith("/rescan")) {
+ InitRescanAndReport(ev);
+ } else if (path.EndsWith("/clean")) {
+ InitCleaningAndReport(ev);
+ } else if (path.EndsWith("/clear_history")) {
+ InitClearHistoryAndReport(ev);
+ } else {
+ RenderMainControlPage(ev);
+ }
+ }
+
+ STATEFN(Working) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TSqsEvents::TEvGarbageSearchResult, ProcessGarbageHints);
+ hFunc(TSqsEvents::TEvGarbageCleaningResult, HandleCleaningResult);
+ hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
+ }
+ }
+
+private:
const TActorId SchemeCacheId;
const TActorId QueuesListReaderId;
- bool Scanning = false;
- THashMap<TString, TGarbageHint> CurrentGarbageHints;
- TVector<TCleaningResult> CleaningHistory;
-};
-
+ bool Scanning = false;
+ THashMap<TString, TGarbageHint> CurrentGarbageHints;
+ TVector<TCleaningResult> CleaningHistory;
+};
+
IActor* CreateGarbageCollector(const TActorId schemeCacheId, const TActorId queuesListReaderId) {
- return new TGarbageCollector(schemeCacheId, queuesListReaderId);
-}
-
-}; // namespace NKikimr::NSQS
+ return new TGarbageCollector(schemeCacheId, queuesListReaderId);
+}
+
+}; // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/garbage_collector.h b/ydb/core/ymq/actor/garbage_collector.h
index d70bbb7491..713b31cc1d 100644
--- a/ydb/core/ymq/actor/garbage_collector.h
+++ b/ydb/core/ymq/actor/garbage_collector.h
@@ -1,10 +1,10 @@
-#pragma once
+#pragma once
#include "defs.h"
-
-#include "actor.h"
-
+
+#include "actor.h"
+
#include <ydb/core/ymq/base/action.h>
-
-namespace NKikimr::NSQS {
+
+namespace NKikimr::NSQS {
IActor* CreateGarbageCollector(const TActorId schemeCacheId, const TActorId queuesListReaderId);
-}
+}
diff --git a/ydb/core/ymq/actor/get_queue_attributes.cpp b/ydb/core/ymq/actor/get_queue_attributes.cpp
index 4cfe7f6402..28dc67b14f 100644
--- a/ydb/core/ymq/actor/get_queue_attributes.cpp
+++ b/ydb/core/ymq/actor/get_queue_attributes.cpp
@@ -20,24 +20,24 @@ namespace NKikimr::NSQS {
struct TAttributeInfo {
bool NeedRuntimeAttributes = false;
bool NeedAttributesTable = false;
- bool NeedArn = false;
+ bool NeedArn = false;
bool FifoOnly = false;
};
static const std::map<TString, TAttributeInfo> AttributesInfo = {
- { "ApproximateNumberOfMessages", { true, false, false, false } },
- { "ApproximateNumberOfMessagesDelayed", { true, false, false, false } },
- { "ApproximateNumberOfMessagesNotVisible", { true, false, false, false } },
- { "CreatedTimestamp", { true, false, false, false } },
- { "DelaySeconds", { false, true, false, false } },
- { "MaximumMessageSize", { false, true, false, false } },
- { "MessageRetentionPeriod", { false, true, false, false } },
- { "ReceiveMessageWaitTimeSeconds", { false, true, false, false } },
- { "RedrivePolicy", { false, true, false, false } },
- { "VisibilityTimeout", { false, true, false, false } },
- { "FifoQueue", { false, true, false, true } },
- { "ContentBasedDeduplication", { false, true, false, true } },
- { "QueueArn", { false, false, true, false } },
+ { "ApproximateNumberOfMessages", { true, false, false, false } },
+ { "ApproximateNumberOfMessagesDelayed", { true, false, false, false } },
+ { "ApproximateNumberOfMessagesNotVisible", { true, false, false, false } },
+ { "CreatedTimestamp", { true, false, false, false } },
+ { "DelaySeconds", { false, true, false, false } },
+ { "MaximumMessageSize", { false, true, false, false } },
+ { "MessageRetentionPeriod", { false, true, false, false } },
+ { "ReceiveMessageWaitTimeSeconds", { false, true, false, false } },
+ { "RedrivePolicy", { false, true, false, false } },
+ { "VisibilityTimeout", { false, true, false, false } },
+ { "FifoQueue", { false, true, false, true } },
+ { "ContentBasedDeduplication", { false, true, false, true } },
+ { "QueueArn", { false, false, true, false } },
};
class TGetQueueAttributesActor
@@ -49,16 +49,16 @@ public:
{
CopyAccountName(Request());
Response_.MutableGetQueueAttributes()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
private:
bool ExpandNames() {
if (!Request().NamesSize()) {
- return false;
- }
-
+ return false;
+ }
+
bool all = false;
for (const auto& name : Request().names()) {
if (name == "All") {
@@ -75,10 +75,10 @@ private:
if (info->second.NeedRuntimeAttributes) {
NeedRuntimeAttributes_ = true;
}
- if (info->second.NeedArn) {
- NeedArn_ = true;
- }
-
+ if (info->second.NeedArn) {
+ NeedArn_ = true;
+ }
+
AttributesSet_.insert(name);
}
}
@@ -92,7 +92,7 @@ private:
}
NeedRuntimeAttributes_ = true;
NeedAttributesTable_ = true;
- NeedArn_ = true;
+ NeedArn_ = true;
}
return true;
}
@@ -102,10 +102,10 @@ private:
return IsIn(AttributesSet_, name);
}
- TString MakeQueueArn(const TString& prefix, const TString& region, const TString& account, const TString& queueName) const {
- return Join(":", prefix, region, account, queueName);
- }
-
+ TString MakeQueueArn(const TString& prefix, const TString& region, const TString& account, const TString& queueName) const {
+ return Join(":", prefix, region, account, queueName);
+ }
+
bool DoValidate() override {
if (!GetQueueName()) {
MakeError(Response_.MutableGetQueueAttributes(), NErrors::MISSING_PARAMETER, "No QueueName parameter.");
@@ -120,11 +120,11 @@ private:
}
void ReplyIfReady() {
- if (WaitCount_ == 0) {
+ if (WaitCount_ == 0) {
SendReplyAndDie();
- }
- }
-
+ }
+ }
+
void DoAction() override {
Become(&TThis::StateFunc);
@@ -132,7 +132,7 @@ private:
SendReplyAndDie();
return;
}
-
+
if (NeedAttributesTable_) {
TExecutorBuilder builder(SelfId(), RequestId_);
builder
@@ -150,17 +150,17 @@ private:
Send(QueueLeader_, MakeHolder<TSqsEvents::TEvGetRuntimeQueueAttributes>(RequestId_));
++WaitCount_;
}
-
- if (NeedArn_) {
- if (IsCloud()) {
+
+ if (NeedArn_) {
+ if (IsCloud()) {
Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvGetQueueFolderIdAndCustomName(RequestId_, UserName_, GetQueueName()));
- ++WaitCount_;
- } else {
- auto* result = Response_.MutableGetQueueAttributes();
+ ++WaitCount_;
+ } else {
+ auto* result = Response_.MutableGetQueueAttributes();
result->SetQueueArn(MakeQueueArn(yaSqsArnPrefix, Cfg().GetYandexCloudServiceRegion(), UserName_, GetQueueName()));
- }
- }
-
+ }
+ }
+
ReplyIfReady();
}
@@ -207,16 +207,16 @@ private:
if (HasAttributeName("VisibilityTimeout")) {
result->SetVisibilityTimeout(TDuration::MilliSeconds(ui64(attrs["VisibilityTimeout"])).Seconds());
}
- if (HasAttributeName("RedrivePolicy")) {
- const TValue& dlqArn(attrs["DlqArn"]);
- if (dlqArn.HaveValue() && !TString(dlqArn).empty()) {
- // the attributes can't be set separately, so we check only one
- TRedrivePolicy redrivePolicy;
- redrivePolicy.TargetArn = TString(dlqArn);
- redrivePolicy.MaxReceiveCount = ui64(attrs["MaxReceiveCount"]);
- result->SetRedrivePolicy(redrivePolicy.ToJson());
- }
- }
+ if (HasAttributeName("RedrivePolicy")) {
+ const TValue& dlqArn(attrs["DlqArn"]);
+ if (dlqArn.HaveValue() && !TString(dlqArn).empty()) {
+ // the attributes can't be set separately, so we check only one
+ TRedrivePolicy redrivePolicy;
+ redrivePolicy.TargetArn = TString(dlqArn);
+ redrivePolicy.MaxReceiveCount = ui64(attrs["MaxReceiveCount"]);
+ result->SetRedrivePolicy(redrivePolicy.ToJson());
+ }
+ }
} else {
RLOG_SQS_ERROR("Get queue attributes query failed");
MakeError(result, NErrors::INTERNAL_FAILURE);
@@ -253,23 +253,23 @@ private:
--WaitCount_;
ReplyIfReady();
- }
-
+ }
+
void HandleQueueFolderIdAndCustomName(TSqsEvents::TEvQueueFolderIdAndCustomName::TPtr& ev) {
- auto* result = Response_.MutableGetQueueAttributes();
-
- if (ev->Get()->Failed || !ev->Get()->Exists) {
+ auto* result = Response_.MutableGetQueueAttributes();
+
+ if (ev->Get()->Failed || !ev->Get()->Exists) {
RLOG_SQS_DEBUG("Get queue folder id and custom name failed. Failed: " << ev->Get()->Failed << ". Exists: " << ev->Get()->Exists);
- MakeError(result, NErrors::INTERNAL_FAILURE);
+ MakeError(result, NErrors::INTERNAL_FAILURE);
SendReplyAndDie();
- return;
+ return;
}
-
- if (NeedArn_) {
+
+ if (NeedArn_) {
result->SetQueueArn(MakeQueueArn(cloudArnPrefix, Cfg().GetYandexCloudServiceRegion(), ev->Get()->QueueFolderId, ev->Get()->QueueCustomName));
- }
-
- --WaitCount_;
+ }
+
+ --WaitCount_;
ReplyIfReady();
}
@@ -281,7 +281,7 @@ private:
THashSet<TString> AttributesSet_;
bool NeedRuntimeAttributes_ = false;
bool NeedAttributesTable_ = false;
- bool NeedArn_ = false;
+ bool NeedArn_ = false;
size_t WaitCount_ = 0;
};
@@ -306,11 +306,11 @@ private:
const auto& entry = Request().GetEntries(i);
auto& req = *ret[i].MutableGetQueueAttributes();
req.MutableAuth()->SetUserName(UserName_);
-
+
if (Request().HasCredentials()) {
*req.MutableCredentials() = Request().GetCredentials();
- }
-
+ }
+
req.SetQueueName(entry.GetQueueName());
req.SetId(entry.GetId());
*req.MutableNames() = Request().GetNames();
diff --git a/ydb/core/ymq/actor/get_queue_url.cpp b/ydb/core/ymq/actor/get_queue_url.cpp
index 5094214104..95554d7bb0 100644
--- a/ydb/core/ymq/actor/get_queue_url.cpp
+++ b/ydb/core/ymq/actor/get_queue_url.cpp
@@ -23,14 +23,14 @@ public:
{
CopyAccountName(Request());
Response_.MutableGetQueueUrl()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
- static constexpr bool NeedExistingQueue() {
- return false;
- }
-
+ static constexpr bool NeedExistingQueue() {
+ return false;
+ }
+
private:
bool DoValidate() override {
if (!GetQueueName()) {
@@ -68,11 +68,11 @@ private:
} else {
if (ev->Get()->Exists) {
auto* result = Response_.MutableGetQueueUrl();
- if (IsCloud()) {
+ if (IsCloud()) {
result->SetQueueUrl(MakeQueueUrl(TString::Join(ev->Get()->QueueId, "/", GetQueueName())));
- } else {
+ } else {
result->SetQueueUrl(MakeQueueUrl(ev->Get()->QueueId));
- }
+ }
} else {
MakeError(MutableErrorDesc(), NErrors::NON_EXISTENT_QUEUE);
}
diff --git a/ydb/core/ymq/actor/infly.cpp b/ydb/core/ymq/actor/infly.cpp
index 2e6bd1e45d..a08cb65387 100644
--- a/ydb/core/ymq/actor/infly.cpp
+++ b/ydb/core/ymq/actor/infly.cpp
@@ -109,11 +109,11 @@ TInflyMessages::TReceiveCandidates::TReceiveCandidates(TIntrusivePtr<TInflyMessa
{
}
-void TInflyMessages::TReceiveCandidates::SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount) {
+void TInflyMessages::TReceiveCandidates::SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount) {
Y_ASSERT(Parent && ReceivedMessages);
if (auto* msg = ReceivedMessages->Find(offset)) {
msg->Message().SetVisibilityDeadline(visibilityDeadline);
- msg->Message().SetReceiveCount(receiveCount);
+ msg->Message().SetReceiveCount(receiveCount);
}
}
diff --git a/ydb/core/ymq/actor/infly.h b/ydb/core/ymq/actor/infly.h
index 571eb3fcd5..1dea6baa6c 100644
--- a/ydb/core/ymq/actor/infly.h
+++ b/ydb/core/ymq/actor/infly.h
@@ -41,11 +41,11 @@ class TInflyMessage : public TInflyMessageWithVisibilityDeadlineKey, public TInf
public:
TInflyMessage() = default;
- TInflyMessage(const ui64 offset, const ui64 randomId, const TInstant visibilityDeadline, const ui32 receiveCount)
+ TInflyMessage(const ui64 offset, const ui64 randomId, const TInstant visibilityDeadline, const ui32 receiveCount)
: Offset(offset)
, RandomId(randomId)
, VisibilityDeadline(visibilityDeadline)
- , ReceiveCount(receiveCount)
+ , ReceiveCount(receiveCount)
{
}
@@ -61,23 +61,23 @@ public:
return VisibilityDeadline;
}
- ui32 GetReceiveCount() const {
- return ReceiveCount;
- }
-
- void SetVisibilityDeadline(const TInstant visibilityDeadline) {
+ ui32 GetReceiveCount() const {
+ return ReceiveCount;
+ }
+
+ void SetVisibilityDeadline(const TInstant visibilityDeadline) {
VisibilityDeadline = visibilityDeadline;
}
- void SetReceiveCount(const ui32 receiveCount) {
- ReceiveCount = receiveCount;
- }
-
+ void SetReceiveCount(const ui32 receiveCount) {
+ ReceiveCount = receiveCount;
+ }
+
private:
ui64 Offset = 0;
ui64 RandomId = 0;
TInstant VisibilityDeadline;
- ui32 ReceiveCount = 0;
+ ui32 ReceiveCount = 0;
};
class TInflyMessages : public TAtomicRefCount<TInflyMessages> {
@@ -109,8 +109,8 @@ public:
return !operator!();
}
- void SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount);
-
+ void SetVisibilityDeadlineAndReceiveCount(ui64 offset, TInstant visibilityDeadline, const ui32 receiveCount);
+
THolder<TInflyMessage> Delete(ui64 offset);
bool Has(ui64 offset) const;
diff --git a/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp b/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp
index 2ccd8613ef..fd4866fd49 100644
--- a/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp
+++ b/ydb/core/ymq/actor/list_dead_letter_source_queues.cpp
@@ -1,114 +1,114 @@
-#include "action.h"
-#include "error.h"
-#include "log.h"
-#include "params.h"
-#include "serviceid.h"
-
+#include "action.h"
+#include "error.h"
+#include "log.h"
+#include "params.h"
+#include "serviceid.h"
+
#include <ydb/public/lib/value/value.h>
#include <ydb/core/ymq/actor/executor.h>
-
-#include <util/string/join.h>
-#include <util/string/printf.h>
-
-using NKikimr::NClient::TValue;
-
-namespace NKikimr::NSQS {
-
-class TListDeadLetterSourceQueuesActor
- : public TActionActor<TListDeadLetterSourceQueuesActor>
-{
-public:
+
+#include <util/string/join.h>
+#include <util/string/printf.h>
+
+using NKikimr::NClient::TValue;
+
+namespace NKikimr::NSQS {
+
+class TListDeadLetterSourceQueuesActor
+ : public TActionActor<TListDeadLetterSourceQueuesActor>
+{
+public:
TListDeadLetterSourceQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb)
: TActionActor(sourceSqsRequest, EAction::ListDeadLetterSourceQueues, std::move(cb))
- {
+ {
CopyAccountName(Request());
- Response_.MutableListDeadLetterSourceQueues()->SetRequestId(RequestId_);
-
+ Response_.MutableListDeadLetterSourceQueues()->SetRequestId(RequestId_);
+
CopySecurityToken(Request());
- }
-
- static constexpr bool NeedExistingQueue() {
- return true;
- }
-
- TString DoGetQueueName() const override {
+ }
+
+ static constexpr bool NeedExistingQueue() {
+ return true;
+ }
+
+ TString DoGetQueueName() const override {
return Request().GetQueueName();
- }
-
-private:
+ }
+
+private:
STATEFN(StateFunc) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TEvWakeup, HandleWakeup);
hFunc(TSqsEvents::TEvExecuted, HandleExecuted);
- }
- }
-
- bool DoValidate() override {
- if (!GetQueueName()) {
- MakeError(MutableErrorDesc(), NErrors::MISSING_PARAMETER, "No QueueName parameter.");
- return false;
- }
-
- return true;
- }
-
- TError* MutableErrorDesc() override {
- return Response_.MutableListDeadLetterSourceQueues()->MutableError();
- }
-
+ }
+ }
+
+ bool DoValidate() override {
+ if (!GetQueueName()) {
+ MakeError(MutableErrorDesc(), NErrors::MISSING_PARAMETER, "No QueueName parameter.");
+ return false;
+ }
+
+ return true;
+ }
+
+ TError* MutableErrorDesc() override {
+ return Response_.MutableListDeadLetterSourceQueues()->MutableError();
+ }
+
void DoAction() override {
- Become(&TThis::StateFunc);
-
+ Become(&TThis::StateFunc);
+
TExecutorBuilder builder(SelfId(), RequestId_);
- builder
- .User(UserName_)
- .Queue(GetQueueName())
+ builder
+ .User(UserName_)
+ .Queue(GetQueueName())
.QueueLeader(QueueLeader_)
- .QueryId(LIST_DEAD_LETTER_SOURCE_QUEUES_ID)
- .Counters(QueueCounters_)
- .RetryOnTimeout()
- .Params()
+ .QueryId(LIST_DEAD_LETTER_SOURCE_QUEUES_ID)
+ .Counters(QueueCounters_)
+ .RetryOnTimeout()
+ .Params()
.Utf8("USER_NAME", UserName_)
- .Utf8("FOLDERID", FolderId_);
-
- builder.Start();
- }
-
+ .Utf8("FOLDERID", FolderId_);
+
+ builder.Start();
+ }
+
void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- auto* result = Response_.MutableListDeadLetterSourceQueues();
-
- if (ev->Get()->IsOk()) {
- const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
- const TValue queues(val["queues"]);
-
- for (size_t i = 0; i < queues.Size(); ++i) {
- const TString name((TString(queues[i]["QueueName"])));
- const TString customQueueName((TString(queues[i]["CustomQueueName"])));
-
- auto* item = result->AddQueues();
- item->SetQueueName(name);
- if (IsCloud()) {
- item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName)));
- } else {
- item->SetQueueUrl(MakeQueueUrl(name));
- }
- }
- } else {
+ const auto& record = ev->Get()->Record;
+ auto* result = Response_.MutableListDeadLetterSourceQueues();
+
+ if (ev->Get()->IsOk()) {
+ const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
+ const TValue queues(val["queues"]);
+
+ for (size_t i = 0; i < queues.Size(); ++i) {
+ const TString name((TString(queues[i]["QueueName"])));
+ const TString customQueueName((TString(queues[i]["CustomQueueName"])));
+
+ auto* item = result->AddQueues();
+ item->SetQueueName(name);
+ if (IsCloud()) {
+ item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName)));
+ } else {
+ item->SetQueueUrl(MakeQueueUrl(name));
+ }
+ }
+ } else {
RLOG_SQS_WARN("Request failed: " << record);
MakeError(result, NErrors::INTERNAL_FAILURE);
- }
-
+ }
+
SendReplyAndDie();
- }
-
+ }
+
const TListDeadLetterSourceQueuesRequest& Request() const {
return SourceSqsRequest_.GetListDeadLetterSourceQueues();
}
-};
-
+};
+
IActor* CreateListDeadLetterSourceQueuesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) {
return new TListDeadLetterSourceQueuesActor(sourceSqsRequest, std::move(cb));
-}
-
-} // namespace NKikimr::NSQS
+}
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/list_permissions.cpp b/ydb/core/ymq/actor/list_permissions.cpp
index 8fcccb084a..93a15e1dbf 100644
--- a/ydb/core/ymq/actor/list_permissions.cpp
+++ b/ydb/core/ymq/actor/list_permissions.cpp
@@ -1,167 +1,167 @@
-#include "action.h"
-
+#include "action.h"
+
#include <ydb/core/protos/flat_tx_scheme.pb.h>
#include <ydb/core/tx/schemeshard/schemeshard.h>
-
+
#include <google/protobuf/text_format.h>
-
-#include <util/string/ascii.h>
-#include <util/string/vector.h>
-#include <util/string/cast.h>
-#include <util/string/join.h>
-
+
+#include <util/string/ascii.h>
+#include <util/string/vector.h>
+#include <util/string/cast.h>
+#include <util/string/join.h>
+
namespace NKikimr::NSQS {
-class TListPermissionsActor
- : public TActionActor<TListPermissionsActor> {
-public:
+class TListPermissionsActor
+ : public TActionActor<TListPermissionsActor> {
+public:
TListPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb)
: TActionActor(sourceSqsRequest, EAction::ListPermissions, std::move(cb))
- {
- Response_.MutableListPermissions()->SetRequestId(RequestId_);
+ {
+ Response_.MutableListPermissions()->SetRequestId(RequestId_);
CopySecurityToken(Request());
- }
-
- static constexpr bool NeedExistingQueue() {
- return false;
- }
-
+ }
+
+ static constexpr bool NeedExistingQueue() {
+ return false;
+ }
+
static constexpr bool NeedUserSpecified() {
return false;
}
-private:
- bool DoValidate() override {
+private:
+ bool DoValidate() override {
if (!Request().GetPath()) {
- MakeError(Response_.MutableListPermissions(), NErrors::MISSING_PARAMETER, "No Path parameter.");
- return false;
- }
-
+ MakeError(Response_.MutableListPermissions(), NErrors::MISSING_PARAMETER, "No Path parameter.");
+ return false;
+ }
+
Path_ = MakeAbsolutePath(Request().GetPath());
-
- if (IsForbiddenPath(Path_)) {
+
+ if (IsForbiddenPath(Path_)) {
MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, Sprintf("Path does not exist: %s.", SanitizeNodePath(Path_).c_str()));
- return false;
- }
-
- return true;
- }
-
- TError* MutableErrorDesc() override {
- return Response_.MutableListPermissions()->MutableError();
- }
-
- virtual TString GetCustomACLPath() const override {
+ return false;
+ }
+
+ return true;
+ }
+
+ TError* MutableErrorDesc() override {
+ return Response_.MutableListPermissions()->MutableError();
+ }
+
+ virtual TString GetCustomACLPath() const override {
return MakeAbsolutePath(Request().GetPath());
- }
-
- TString DoGetQueueName() const override {
- return {};
- }
-
+ }
+
+ TString DoGetQueueName() const override {
+ return {};
+ }
+
void RequestSchemeShard(const TString& path) {
- std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate());
+ std::unique_ptr<TEvTxUserProxy::TEvNavigate> navigateRequest(new TEvTxUserProxy::TEvNavigate());
NKikimrSchemeOp::TDescribePath* record = navigateRequest->Record.MutableDescribePath();
- record->SetPath(path);
-
+ record->SetPath(path);
+
Send(MakeTxProxyID(), navigateRequest.release());
- }
-
+ }
+
void DoAction() override {
- Become(&TThis::WaitSchemeShardResponse);
+ Become(&TThis::WaitSchemeShardResponse);
RequestSchemeShard(Path_);
- }
-
+ }
+
STATEFN(WaitSchemeShardResponse) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TEvWakeup, HandleWakeup);
hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, HandleSchemeShardResponse);
- }
- }
-
- using TGroupedACE = THashMap<TString, TSet<TStringBuf>>;
-
- TGroupedACE GroupACEBySID(const NACLibProto::TACL& acl) const {
- TGroupedACE result;
- for (const NACLibProto::TACE& ace : acl.GetACE()) {
- if (static_cast<NACLib::EAccessType>(ace.GetAccessType()) == NACLib::EAccessType::Allow) {
- if (ace.GetSID()) {
- auto& acesForSIDSet = result[ace.GetSID()];
- const auto aces = GetAccessMatchingACE(ace.GetAccessRight());
- for (const TStringBuf& aceName : aces) {
- acesForSIDSet.insert(aceName);
- }
- }
- }
- }
-
- return result;
- }
-
- template<typename TMutablePermissions>
- void ConvertACLToPermissions(const NACLibProto::TACL& acl, TMutablePermissions& nodePermissions) const {
- const auto acesBySID = GroupACEBySID(acl);
- for (auto&& pair : acesBySID) {
- const auto& SID = pair.first;
- const auto& acesSet = pair.second;
-
- auto* permissions = nodePermissions.Add();
-
- permissions->SetSubject(SID);
- for (const auto& aceName : acesSet) {
- permissions->MutablePermissionNames()->Add(TString(aceName));
- }
- }
- }
-
+ }
+ }
+
+ using TGroupedACE = THashMap<TString, TSet<TStringBuf>>;
+
+ TGroupedACE GroupACEBySID(const NACLibProto::TACL& acl) const {
+ TGroupedACE result;
+ for (const NACLibProto::TACE& ace : acl.GetACE()) {
+ if (static_cast<NACLib::EAccessType>(ace.GetAccessType()) == NACLib::EAccessType::Allow) {
+ if (ace.GetSID()) {
+ auto& acesForSIDSet = result[ace.GetSID()];
+ const auto aces = GetAccessMatchingACE(ace.GetAccessRight());
+ for (const TStringBuf& aceName : aces) {
+ acesForSIDSet.insert(aceName);
+ }
+ }
+ }
+ }
+
+ return result;
+ }
+
+ template<typename TMutablePermissions>
+ void ConvertACLToPermissions(const NACLibProto::TACL& acl, TMutablePermissions& nodePermissions) const {
+ const auto acesBySID = GroupACEBySID(acl);
+ for (auto&& pair : acesBySID) {
+ const auto& SID = pair.first;
+ const auto& acesSet = pair.second;
+
+ auto* permissions = nodePermissions.Add();
+
+ permissions->SetSubject(SID);
+ for (const auto& aceName : acesSet) {
+ permissions->MutablePermissionNames()->Add(TString(aceName));
+ }
+ }
+ }
+
void HandleSchemeShardResponse(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) {
- const auto& record = ev->Get()->GetRecord();
- const auto status = record.GetStatus();
- switch (status) {
+ const auto& record = ev->Get()->GetRecord();
+ const auto status = record.GetStatus();
+ switch (status) {
case NKikimrScheme::StatusSuccess: {
- const auto& pathDescription = record.GetPathDescription();
+ const auto& pathDescription = record.GetPathDescription();
const NKikimrSchemeOp::TDirEntry& entry = pathDescription.GetSelf();
-
- TSecurityObject secObjWithACL(entry.GetOwner(), entry.GetACL(), false);
- TSecurityObject secObjWithEffectiveACL(entry.GetOwner(), entry.GetEffectiveACL(), false);
-
- auto convertACLToPermissions = [this, &secObjWithACL, &secObjWithEffectiveACL](auto* mutableNodePermissions) {
- ConvertACLToPermissions(secObjWithACL.GetACL(), *mutableNodePermissions->MutablePermissions());
- ConvertACLToPermissions(secObjWithEffectiveACL.GetACL(), *mutableNodePermissions->MutableEffectivePermissions());
- };
-
- const size_t depth = CalculatePathDepth(SanitizeNodePath(Path_));
- if (depth < 2) {
- convertACLToPermissions(Response_.MutableListPermissions()->MutableAccountPermissions());
- } else {
- convertACLToPermissions(Response_.MutableListPermissions()->MutableQueuePermissions());
- }
-
- break;
- }
+
+ TSecurityObject secObjWithACL(entry.GetOwner(), entry.GetACL(), false);
+ TSecurityObject secObjWithEffectiveACL(entry.GetOwner(), entry.GetEffectiveACL(), false);
+
+ auto convertACLToPermissions = [this, &secObjWithACL, &secObjWithEffectiveACL](auto* mutableNodePermissions) {
+ ConvertACLToPermissions(secObjWithACL.GetACL(), *mutableNodePermissions->MutablePermissions());
+ ConvertACLToPermissions(secObjWithEffectiveACL.GetACL(), *mutableNodePermissions->MutableEffectivePermissions());
+ };
+
+ const size_t depth = CalculatePathDepth(SanitizeNodePath(Path_));
+ if (depth < 2) {
+ convertACLToPermissions(Response_.MutableListPermissions()->MutableAccountPermissions());
+ } else {
+ convertACLToPermissions(Response_.MutableListPermissions()->MutableQueuePermissions());
+ }
+
+ break;
+ }
case NKikimrScheme::StatusPathDoesNotExist: {
MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, Sprintf("Path does not exist: %s.", SanitizeNodePath(Path_).c_str()));
- break;
- }
- default: {
- MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
- break;
- }
- }
-
+ break;
+ }
+ default: {
+ MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
+ break;
+ }
+ }
+
SendReplyAndDie();
- }
-
+ }
+
const TListPermissionsRequest& Request() const {
return SourceSqsRequest_.GetListPermissions();
}
-private:
- TString Path_;
-};
-
+private:
+ TString Path_;
+};
+
IActor* CreateListPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) {
return new TListPermissionsActor(sourceSqsRequest, std::move(cb));
-}
-
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/list_queues.cpp b/ydb/core/ymq/actor/list_queues.cpp
index c9cba83f4e..25ae108376 100644
--- a/ydb/core/ymq/actor/list_queues.cpp
+++ b/ydb/core/ymq/actor/list_queues.cpp
@@ -1,7 +1,7 @@
#include "action.h"
#include "error.h"
#include "log.h"
-#include "params.h"
+#include "params.h"
#include "serviceid.h"
#include "executor.h"
@@ -28,7 +28,7 @@ public:
{
CopyAccountName(Request());
Response_.MutableListQueues()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
@@ -50,19 +50,19 @@ private:
}
void DoAction() override {
- Become(&TThis::StateFunc);
-
+ Become(&TThis::StateFunc);
+
if (!UserExists_) {
if (!IsCloud()) {
MakeError(Response_.MutableListQueues(), NErrors::OPT_IN_REQUIRED, "The specified account does not exist.");
} // else respond with an empty list for inexistent account
SendReplyAndDie();
- return;
- }
-
+ return;
+ }
+
DiscoverQueues();
- }
-
+ }
+
TString DoGetQueueName() const override {
return TString();
}
@@ -86,16 +86,16 @@ private:
for (size_t i = 0; i < queues.Size(); ++i) {
const TString name((TString(queues[i]["QueueName"])));
- const TString customQueueName((TString(queues[i]["CustomQueueName"])));
+ const TString customQueueName((TString(queues[i]["CustomQueueName"])));
- if (prefix.empty() || AsciiHasPrefix((IsCloud() ? customQueueName : name), prefix)) {
+ if (prefix.empty() || AsciiHasPrefix((IsCloud() ? customQueueName : name), prefix)) {
auto* item = result->AddQueues();
item->SetQueueName(name);
- if (IsCloud()) {
- item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName)));
- } else {
- item->SetQueueUrl(MakeQueueUrl(name));
- }
+ if (IsCloud()) {
+ item->SetQueueUrl(MakeQueueUrl(TString::Join(name, '/', customQueueName)));
+ } else {
+ item->SetQueueUrl(MakeQueueUrl(name));
+ }
}
}
} else {
diff --git a/ydb/core/ymq/actor/list_users.cpp b/ydb/core/ymq/actor/list_users.cpp
index 0cc23a1f92..d653fc20a3 100644
--- a/ydb/core/ymq/actor/list_users.cpp
+++ b/ydb/core/ymq/actor/list_users.cpp
@@ -24,7 +24,7 @@ public:
{
CopyAccountName(Request());
Response_.MutableListUsers()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
diff --git a/ydb/core/ymq/actor/metering.cpp b/ydb/core/ymq/actor/metering.cpp
index 17b32be63a..94ccef7eb0 100644
--- a/ydb/core/ymq/actor/metering.cpp
+++ b/ydb/core/ymq/actor/metering.cpp
@@ -1,29 +1,29 @@
-#include "metering.h"
-
-#include "cfg.h"
-#include "serviceid.h"
-#include "proxy_actor.h"
-
+#include "metering.h"
+
+#include "cfg.h"
+#include "serviceid.h"
+#include "proxy_actor.h"
+
#include <ydb/core/mind/address_classification/net_classifier.h>
#include <ydb/core/ymq/base/action.h>
#include <ydb/core/ymq/base/counters.h>
#include <ydb/core/ymq/base/processed_request_attributes.h>
-
+
#include <library/cpp/logger/global/global.h>
#include <library/cpp/logger/record.h>
-
-#include <util/generic/guid.h>
+
+#include <util/generic/guid.h>
#include <util/generic/serialized_enum.h>
-#include <util/string/builder.h>
-#include <util/system/hostname.h>
-
-#include <tuple>
-
-namespace NKikimr::NSQS {
-
-using namespace NAddressClassifier;
-
+#include <util/string/builder.h>
+#include <util/system/hostname.h>
+
+#include <tuple>
+
+namespace NKikimr::NSQS {
+
+using namespace NAddressClassifier;
+
const THashMap<TString, TString> TProcessedRequestsAggregator::LabelTransformation = {
{"yacloud", ToString(ENetworkClass::cloud)}
};
@@ -42,17 +42,17 @@ TProcessedRequestsAggregator::TProcessedRequestsAggregator(const NKikimrConfig::
InitAddressClassifier(config, std::move(labels));
}
}
-
-bool TProcessedRequestsAggregator::TReportedTrafficKey::operator<(const TReportedTrafficKey& rhs) const {
+
+bool TProcessedRequestsAggregator::TReportedTrafficKey::operator<(const TReportedTrafficKey& rhs) const {
return std::tie(ResourceId, TrafficType, NetworkClassLabel) <
std::tie(rhs.ResourceId, rhs.TrafficType, rhs.NetworkClassLabel);
-}
-
-bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReportedRequestsKey& rhs) const {
- return std::tie(ResourceId, QueueType) <
- std::tie(rhs.ResourceId, rhs.QueueType);
-}
-
+}
+
+bool TProcessedRequestsAggregator::TReportedRequestsKey::operator<(const TReportedRequestsKey& rhs) const {
+ return std::tie(ResourceId, QueueType) <
+ std::tie(rhs.ResourceId, rhs.QueueType);
+}
+
TString TProcessedRequestsAggregator::ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address) {
if (classifier) {
auto result = classifier->ClassifyAddress(address);
@@ -77,41 +77,41 @@ TString TProcessedRequestsAggregator::ClassifyNetwork(const TString& address) co
} else {
result = ClassifyNetwork(AddressClassifier, address);
INC_COUNTER(Counters, IdleClassifierRequestsResults[netClassifierResult]);
- }
+ }
INC_COUNTER(Counters, ClassifierRequestsResults[result]);
return result;
-}
-
-ui64 TProcessedRequestsAggregator::CountBlocks(const ui64 bytes, const ui64 blockSize) const {
- return (bytes - 1) / blockSize + 1;
-}
-
-bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs) {
- if (attrs.HttpStatusCode >= 500 || !attrs.FolderId || !attrs.SourceAddress) {
- return false; // do not log with insufficient attributes
- }
-
- TString resourceId = attrs.ResourceId;
- EQueueType queueType = attrs.IsFifo ? EQueueType::fifo : EQueueType::std;
- if (!IsActionForQueue(attrs.Action) || attrs.Action == EAction::GetQueueUrl) {
- queueType = EQueueType::other;
- resourceId = "";
- }
+}
+
+ui64 TProcessedRequestsAggregator::CountBlocks(const ui64 bytes, const ui64 blockSize) const {
+ return (bytes - 1) / blockSize + 1;
+}
+
+bool TProcessedRequestsAggregator::Add(const TProcessedRequestAttributes& attrs) {
+ if (attrs.HttpStatusCode >= 500 || !attrs.FolderId || !attrs.SourceAddress) {
+ return false; // do not log with insufficient attributes
+ }
+
+ TString resourceId = attrs.ResourceId;
+ EQueueType queueType = attrs.IsFifo ? EQueueType::fifo : EQueueType::std;
+ if (!IsActionForQueue(attrs.Action) || attrs.Action == EAction::GetQueueUrl) {
+ queueType = EQueueType::other;
+ resourceId = "";
+ }
const auto networkClassLabel = ClassifyNetwork(attrs.SourceAddress);
-
- Y_VERIFY(attrs.RequestSizeInBytes);
- Y_VERIFY(attrs.ResponseSizeInBytes);
-
+
+ Y_VERIFY(attrs.RequestSizeInBytes);
+ Y_VERIFY(attrs.ResponseSizeInBytes);
+
ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::ingress, networkClassLabel}] += attrs.RequestSizeInBytes;
ReportedTraffic[attrs.FolderId][{resourceId, ETrafficType::egress, networkClassLabel}] += attrs.ResponseSizeInBytes;
-
- static const ui64 defaultRequestBlockSize = 64 * 1024; // SQS-22
- ReportedRequests[attrs.FolderId][{resourceId, queueType}] += CountBlocks(attrs.RequestSizeInBytes + attrs.ResponseSizeInBytes,
- defaultRequestBlockSize);
-
- return true;
-}
-
+
+ static const ui64 defaultRequestBlockSize = 64 * 1024; // SQS-22
+ ReportedRequests[attrs.FolderId][{resourceId, queueType}] += CountBlocks(attrs.RequestSizeInBytes + attrs.ResponseSizeInBytes,
+ defaultRequestBlockSize);
+
+ return true;
+}
+
NSc::TValue CreateMeteringBillingRecord(
const TString& folderId,
const TString& resourceId,
@@ -122,87 +122,87 @@ NSc::TValue CreateMeteringBillingRecord(
const TString& unit,
const NSc::TValue& tags
) {
- const TString& billingRecordVersion = "v1";
- const ui64 utcSeconds = now.Seconds();
-
- NSc::TValue record;
- record["folder_id"] = folderId;
- record["id"] = CreateGuidAsString();
- if (resourceId) {
- record["resource_id"] = resourceId;
- }
- record["schema"] = schema;
- record["source_id"] = fqdn;
- record["source_wt"] = utcSeconds;
-
- auto& usage = record["usage"].SetDict();
- usage["start"] = utcSeconds;
- usage["finish"] = utcSeconds;
- usage["type"] = "delta";
- usage["unit"] = unit;
- usage["quantity"] = quantity;
-
- record["tags"] = tags;
-
- record["version"] = billingRecordVersion;
-
- return record;
-}
-
-TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const {
- TVector<NSc::TValue> result;
-
- for (const auto& perFolderData : ReportedTraffic) {
- const TString& folderId = perFolderData.first;
- for (const auto& [trafficKey, trafficValue] : perFolderData.second) {
- NSc::TValue tags;
- tags.SetDict();
+ const TString& billingRecordVersion = "v1";
+ const ui64 utcSeconds = now.Seconds();
+
+ NSc::TValue record;
+ record["folder_id"] = folderId;
+ record["id"] = CreateGuidAsString();
+ if (resourceId) {
+ record["resource_id"] = resourceId;
+ }
+ record["schema"] = schema;
+ record["source_id"] = fqdn;
+ record["source_wt"] = utcSeconds;
+
+ auto& usage = record["usage"].SetDict();
+ usage["start"] = utcSeconds;
+ usage["finish"] = utcSeconds;
+ usage["type"] = "delta";
+ usage["unit"] = unit;
+ usage["quantity"] = quantity;
+
+ record["tags"] = tags;
+
+ record["version"] = billingRecordVersion;
+
+ return record;
+}
+
+TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const {
+ TVector<NSc::TValue> result;
+
+ for (const auto& perFolderData : ReportedTraffic) {
+ const TString& folderId = perFolderData.first;
+ for (const auto& [trafficKey, trafficValue] : perFolderData.second) {
+ NSc::TValue tags;
+ tags.SetDict();
tags["type"] = trafficKey.NetworkClassLabel;
- tags["direction"] = ToString(trafficKey.TrafficType);
-
+ tags["direction"] = ToString(trafficKey.TrafficType);
+
result.push_back(CreateMeteringBillingRecord(folderId,
- trafficKey.ResourceId,
- "ymq.traffic.v1",
- fqdn,
- now,
- trafficValue,
- "byte",
- tags));
- }
- }
-
- return result;
-}
-
-TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const {
- TVector<NSc::TValue> result;
-
- for (const auto& perFolderData : ReportedRequests) {
- const TString& folderId = perFolderData.first;
- for (const auto& [requestsKey, requestsValue] : perFolderData.second) {
- NSc::TValue tags;
- tags.SetDict();
- tags["queue_type"] = ToString(requestsKey.QueueType);
-
+ trafficKey.ResourceId,
+ "ymq.traffic.v1",
+ fqdn,
+ now,
+ trafficValue,
+ "byte",
+ tags));
+ }
+ }
+
+ return result;
+}
+
+TVector<NSc::TValue> TProcessedRequestsAggregator::DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const {
+ TVector<NSc::TValue> result;
+
+ for (const auto& perFolderData : ReportedRequests) {
+ const TString& folderId = perFolderData.first;
+ for (const auto& [requestsKey, requestsValue] : perFolderData.second) {
+ NSc::TValue tags;
+ tags.SetDict();
+ tags["queue_type"] = ToString(requestsKey.QueueType);
+
result.push_back(CreateMeteringBillingRecord(folderId,
- requestsKey.ResourceId,
- "ymq.requests.v1",
- fqdn,
- now,
- requestsValue,
- "request",
- tags));
- }
- }
-
- return result;
-}
-
-void TProcessedRequestsAggregator::ResetReportsStorage() {
- ReportedTraffic.clear();
- ReportedRequests.clear();
-}
-
+ requestsKey.ResourceId,
+ "ymq.requests.v1",
+ fqdn,
+ now,
+ requestsValue,
+ "request",
+ tags));
+ }
+ }
+
+ return result;
+}
+
+void TProcessedRequestsAggregator::ResetReportsStorage() {
+ ReportedTraffic.clear();
+ ReportedRequests.clear();
+}
+
void TProcessedRequestsAggregator::UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> /*netDataUpdateTimestamp*/) {
NetClassifier = classifier;
}
@@ -220,87 +220,87 @@ void TProcessedRequestsAggregator::InitAddressClassifier(const NKikimrConfig::TS
AddressClassifier = TLabeledAddressClassifier::MakeLabeledAddressClassifier(std::move(addressClassifier), std::move(labels));
}
-class TMeteringActor
+class TMeteringActor
: public TActorBootstrapped<TMeteringActor>
-{
-public:
- TMeteringActor()
- : HostFQDN(FQDNHostName())
- {
- }
-
+{
+public:
+ TMeteringActor()
+ : HostFQDN(FQDNHostName())
+ {
+ }
+
void Bootstrap(const TActorContext& ctx) {
- Become(&TThis::Work);
-
+ Become(&TThis::Work);
+
ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
Aggregator = MakeHolder<TProcessedRequestsAggregator>(Cfg());
-
+
FlushProcessedRequestsAttributes();
- }
-
- void WriteLogRecord(const NSc::TValue& record, TStringBuilder& records) const {
- records << record.ToJsonSafe() << "\n";
- }
-
+ }
+
+ void WriteLogRecord(const NSc::TValue& record, TStringBuilder& records) const {
+ records << record.ToJsonSafe() << "\n";
+ }
+
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::SQS_METERING_ACTOR;
- }
-
+ }
+
void FlushProcessedRequestsAttributes() {
- Y_VERIFY(Aggregator);
-
+ Y_VERIFY(Aggregator);
+
Schedule(TDuration::MilliSeconds(Cfg().GetMeteringFlushingIntervalMs()), new TEvWakeup());
-
+
const auto reportedTrafficRecords = Aggregator->DumpReportedTrafficAsJsonArray(HostFQDN, TActivationContext::Now());
const auto reportedRequestsRecords = Aggregator->DumpReportedRequestsAsJsonArray(HostFQDN, TActivationContext::Now());
-
+
if (Cfg().GetMeteringLogFilePath()) {
- TStringBuilder records;
-
- for (const auto& trafficRecord : reportedTrafficRecords) {
- WriteLogRecord(trafficRecord, records);
- }
- for (const auto& requestRecord : reportedRequestsRecords) {
- WriteLogRecord(requestRecord, records);
- }
-
- if (const TString packedRecords = records) {
- TLoggerOperator<TGlobalLog>::Log().Write(packedRecords.Data(), packedRecords.Size());
- }
- }
-
- Aggregator->ResetReportsStorage();
- }
-
+ TStringBuilder records;
+
+ for (const auto& trafficRecord : reportedTrafficRecords) {
+ WriteLogRecord(trafficRecord, records);
+ }
+ for (const auto& requestRecord : reportedRequestsRecords) {
+ WriteLogRecord(requestRecord, records);
+ }
+
+ if (const TString packedRecords = records) {
+ TLoggerOperator<TGlobalLog>::Log().Write(packedRecords.Data(), packedRecords.Size());
+ }
+ }
+
+ Aggregator->ResetReportsStorage();
+ }
+
void HandleWakeup(TEvWakeup::TPtr&) {
FlushProcessedRequestsAttributes();
- }
-
+ }
+
void HandleReportProcessedRequestAttributes(TSqsEvents::TEvReportProcessedRequestAttributes::TPtr& ev) {
- Y_VERIFY(Aggregator);
-
- Aggregator->Add(ev->Get()->Data);
- }
-
+ Y_VERIFY(Aggregator);
+
+ Aggregator->Add(ev->Get()->Data);
+ }
+
STATEFN(Work) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TEvWakeup, HandleWakeup);
hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleNetClassifierUpdate);
hFunc(TSqsEvents::TEvReportProcessedRequestAttributes, HandleReportProcessedRequestAttributes);
- }
- }
+ }
+ }
-private:
+private:
void HandleNetClassifierUpdate(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
Aggregator->UpdateNetClassifier(ev->Get()->Classifier, ev->Get()->NetDataUpdateTimestamp);
}
private:
- const TString HostFQDN;
- THolder<TProcessedRequestsAggregator> Aggregator;
-};
-
+ const TString HostFQDN;
+ THolder<TProcessedRequestsAggregator> Aggregator;
+};
+
IActor* CreateSqsMeteringService() { return new TMeteringActor(); }
-
-} // namespace NKikimr::NSQS
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/metering.h b/ydb/core/ymq/actor/metering.h
index e6d5f8da1c..c9d5a58991 100644
--- a/ydb/core/ymq/actor/metering.h
+++ b/ydb/core/ymq/actor/metering.h
@@ -1,20 +1,20 @@
-#pragma once
+#pragma once
#include "defs.h"
-
+
#include <ydb/core/ymq/base/action.h>
#include <ydb/core/util/address_classifier.h>
#include <ydb/core/ymq/base/processed_request_attributes.h>
-
+
#include <ydb/core/protos/config.pb.h>
-
+
#include <library/cpp/scheme/scheme.h>
-
-#include <util/datetime/base.h>
-#include <util/generic/hash.h>
-#include <util/generic/map.h>
-
-namespace NKikimr::NSQS {
-
+
+#include <util/datetime/base.h>
+#include <util/generic/hash.h>
+#include <util/generic/map.h>
+
+namespace NKikimr::NSQS {
+
struct TMeteringCounters;
NSc::TValue CreateMeteringBillingRecord(
@@ -28,60 +28,60 @@ NSc::TValue CreateMeteringBillingRecord(
const NSc::TValue& tags
);
-class TProcessedRequestsAggregator {
-public:
+class TProcessedRequestsAggregator {
+public:
static const THashMap<TString, TString> LabelTransformation;
public:
- TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config);
-
- // Do not rename enum members in the enums below (SQS-22)
- enum ETrafficType {
- ingress = 0,
- egress
- };
-
- enum ENetworkClass {
- inet = 0,
- cloud,
+ TProcessedRequestsAggregator(const NKikimrConfig::TSqsConfig& config);
+
+ // Do not rename enum members in the enums below (SQS-22)
+ enum ETrafficType {
+ ingress = 0,
+ egress
+ };
+
+ enum ENetworkClass {
+ inet = 0,
+ cloud,
yandex,
unknown
- };
-
- enum EQueueType {
- std = 0,
- fifo,
- other
- };
-
- struct TReportedTrafficKey {
- TString ResourceId;
- ETrafficType TrafficType;
+ };
+
+ enum EQueueType {
+ std = 0,
+ fifo,
+ other
+ };
+
+ struct TReportedTrafficKey {
+ TString ResourceId;
+ ETrafficType TrafficType;
TString NetworkClassLabel;
-
- bool operator<(const TReportedTrafficKey& rhs) const;
- };
-
- struct TReportedRequestsKey {
- TString ResourceId;
- EQueueType QueueType;
-
- bool operator<(const TReportedRequestsKey& rhs) const;
- };
-
+
+ bool operator<(const TReportedTrafficKey& rhs) const;
+ };
+
+ struct TReportedRequestsKey {
+ TString ResourceId;
+ EQueueType QueueType;
+
+ bool operator<(const TReportedRequestsKey& rhs) const;
+ };
+
TString ClassifyNetwork(const TString& address) const;
-
- ui64 CountBlocks(const ui64 bytes, const ui64 blockSize) const;
-
- bool Add(const TProcessedRequestAttributes& attrs);
-
- TVector<NSc::TValue> DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const;
- TVector<NSc::TValue> DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const;
-
- void ResetReportsStorage();
-
+
+ ui64 CountBlocks(const ui64 bytes, const ui64 blockSize) const;
+
+ bool Add(const TProcessedRequestAttributes& attrs);
+
+ TVector<NSc::TValue> DumpReportedTrafficAsJsonArray(const TString& fqdn, const TInstant& now) const;
+ TVector<NSc::TValue> DumpReportedRequestsAsJsonArray(const TString& fqdn, const TInstant& now) const;
+
+ void ResetReportsStorage();
+
void UpdateNetClassifier(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, TMaybe<TInstant> netDataUpdateTimestamp);
-private:
+private:
static TString ClassifyNetwork(NAddressClassifier::TLabeledAddressClassifier::TConstPtr classifier, const TString& address);
void InitAddressClassifier(const NKikimrConfig::TSqsConfig& config, TVector<TString>&& labels);
@@ -89,12 +89,12 @@ private:
private:
bool NetClassifierOnly;
- THashMap<TString, TMap<TReportedTrafficKey, ui64>> ReportedTraffic;
- THashMap<TString, TMap<TReportedRequestsKey, ui64>> ReportedRequests;
+ THashMap<TString, TMap<TReportedTrafficKey, ui64>> ReportedTraffic;
+ THashMap<TString, TMap<TReportedRequestsKey, ui64>> ReportedRequests;
NAddressClassifier::TLabeledAddressClassifier::TConstPtr AddressClassifier;
NAddressClassifier::TLabeledAddressClassifier::TConstPtr NetClassifier;
TIntrusivePtr<TMeteringCounters> Counters;
-};
-
-} // namespace NKikimr::NSQS
+};
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/migration.cpp b/ydb/core/ymq/actor/migration.cpp
index aafb5ef35b..e547e97558 100644
--- a/ydb/core/ymq/actor/migration.cpp
+++ b/ydb/core/ymq/actor/migration.cpp
@@ -350,7 +350,7 @@ void TQueueMigrationActor::StartAltering() {
CheckAddColumn("Attributes", "DlqArn", NScheme::NTypeIds::Utf8);
CheckAddColumn("Attributes", "MaxReceiveCount", NScheme::NTypeIds::Uint64);
CheckAddColumn("Attributes", "ShowDetailedCountersDeadline", NScheme::NTypeIds::Uint64);
-
+
CheckAddColumn("State", "InflyVersion", NScheme::NTypeIds::Uint64);
if (!IsFifoQueue) {
for (ui64 shard = 0; shard < ShardsCount; ++shard) {
diff --git a/ydb/core/ymq/actor/modify_permissions.cpp b/ydb/core/ymq/actor/modify_permissions.cpp
index 87207f63e5..8b9e1cb2d7 100644
--- a/ydb/core/ymq/actor/modify_permissions.cpp
+++ b/ydb/core/ymq/actor/modify_permissions.cpp
@@ -1,174 +1,174 @@
-#include "action.h"
-
+#include "action.h"
+
#include <google/protobuf/text_format.h>
-
-#include <util/string/ascii.h>
-#include <util/string/vector.h>
-#include <util/string/cast.h>
-#include <util/string/join.h>
-
+
+#include <util/string/ascii.h>
+#include <util/string/vector.h>
+#include <util/string/cast.h>
+#include <util/string/join.h>
+
namespace NKikimr::NSQS {
-class TModifyPermissionsActor
- : public TActionActor<TModifyPermissionsActor> {
-public:
+class TModifyPermissionsActor
+ : public TActionActor<TModifyPermissionsActor> {
+public:
TModifyPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb)
: TActionActor(sourceSqsRequest, EAction::ModifyPermissions, std::move(cb))
- {
- Response_.MutableModifyPermissions()->SetRequestId(RequestId_);
+ {
+ Response_.MutableModifyPermissions()->SetRequestId(RequestId_);
CopySecurityToken(Request());
- }
-
- static constexpr bool NeedExistingQueue() {
- return false;
- }
-
+ }
+
+ static constexpr bool NeedExistingQueue() {
+ return false;
+ }
+
static constexpr bool NeedUserSpecified() {
return false;
}
-private:
- bool TryConvertSQSPermissionToYdbACLMask(const TString& permission, ui32& mask) const {
- mask = GetACERequiredAccess(permission);
- return mask != 0;
- }
-
-#define PROCESS_MODIFY_ACL_ACTION(PERMISSION_NAME, ACCESS_ACTION_TYPE, CLEAR_FLAG) \
- case TPermissionsAction::Y_CAT(k, PERMISSION_NAME): { \
- const auto& SID = action.Y_CAT(Get, PERMISSION_NAME)().GetSubject(); \
- if (CLEAR_FLAG) { \
- ACLDiff_.ClearAccessForSid(SID); \
- } \
- for (const auto& name : action.Y_CAT(Get, PERMISSION_NAME)().GetPermissionNames()) { \
- if (!TryConvertSQSPermissionToYdbACLMask(name, mask)) { \
- TString permissionName(Y_STRINGIZE(PERMISSION_NAME)); \
- permissionName.to_lower(); \
+private:
+ bool TryConvertSQSPermissionToYdbACLMask(const TString& permission, ui32& mask) const {
+ mask = GetACERequiredAccess(permission);
+ return mask != 0;
+ }
+
+#define PROCESS_MODIFY_ACL_ACTION(PERMISSION_NAME, ACCESS_ACTION_TYPE, CLEAR_FLAG) \
+ case TPermissionsAction::Y_CAT(k, PERMISSION_NAME): { \
+ const auto& SID = action.Y_CAT(Get, PERMISSION_NAME)().GetSubject(); \
+ if (CLEAR_FLAG) { \
+ ACLDiff_.ClearAccessForSid(SID); \
+ } \
+ for (const auto& name : action.Y_CAT(Get, PERMISSION_NAME)().GetPermissionNames()) { \
+ if (!TryConvertSQSPermissionToYdbACLMask(name, mask)) { \
+ TString permissionName(Y_STRINGIZE(PERMISSION_NAME)); \
+ permissionName.to_lower(); \
const auto errorMsg = Sprintf("ModifyPermissions failed to %s unknown permission %s.", permissionName.c_str(), name.c_str()); \
- MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, errorMsg); \
- return false; \
- } \
- ACLDiff_.Y_CAT(ACCESS_ACTION_TYPE, Access)(NACLib::EAccessType::Allow, mask, SID); \
- } \
- break; \
- }
-
- bool DoValidate() override {
+ MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, errorMsg); \
+ return false; \
+ } \
+ ACLDiff_.Y_CAT(ACCESS_ACTION_TYPE, Access)(NACLib::EAccessType::Allow, mask, SID); \
+ } \
+ break; \
+ }
+
+ bool DoValidate() override {
if (!Request().GetResource()) {
- MakeError(Response_.MutableModifyPermissions(), NErrors::MISSING_PARAMETER, "No Resource parameter.");
- return false;
- }
-
+ MakeError(Response_.MutableModifyPermissions(), NErrors::MISSING_PARAMETER, "No Resource parameter.");
+ return false;
+ }
+
Resource_ = MakeAbsolutePath(Request().GetResource());
-
- if (IsForbiddenPath(Resource_)) {
+
+ if (IsForbiddenPath(Resource_)) {
MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, Sprintf("Path does not exist: %s.", SanitizeNodePath(Resource_).c_str()));
- return false;
- }
-
+ return false;
+ }
+
if (Request().GetClearACL()) {
- ACLDiff_.ClearAccess();
- }
-
- ui32 mask = 0;
+ ACLDiff_.ClearAccess();
+ }
+
+ ui32 mask = 0;
for (const auto& action : Request().GetActions()) {
- switch (action.Action_case()) {
- PROCESS_MODIFY_ACL_ACTION(Set, Add, true);
- PROCESS_MODIFY_ACL_ACTION(Grant, Add, false);
- PROCESS_MODIFY_ACL_ACTION(Revoke, Remove, false);
- default: {
+ switch (action.Action_case()) {
+ PROCESS_MODIFY_ACL_ACTION(Set, Add, true);
+ PROCESS_MODIFY_ACL_ACTION(Grant, Add, false);
+ PROCESS_MODIFY_ACL_ACTION(Revoke, Remove, false);
+ default: {
MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, "Unknown ModifyPermissions action.");
- return false;
- }
- }
- }
-
- return true;
- }
-
- TError* MutableErrorDesc() override {
- return Response_.MutableModifyPermissions()->MutableError();
- }
-
- virtual TString GetCustomACLPath() const override {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ TError* MutableErrorDesc() override {
+ return Response_.MutableModifyPermissions()->MutableError();
+ }
+
+ virtual TString GetCustomACLPath() const override {
return MakeAbsolutePath(Request().GetResource());
- }
-
- TString DoGetQueueName() const override {
- return {};
- }
-
+ }
+
+ TString DoGetQueueName() const override {
+ return {};
+ }
+
void DoAction() override {
- Become(&TThis::StateFunc);
-
- TStringBuf workingDir, resource;
- TStringBuf(Resource_).RSplit('/', workingDir, resource);
-
- auto proposeRequest = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record;
+ Become(&TThis::StateFunc);
+
+ TStringBuf workingDir, resource;
+ TStringBuf(Resource_).RSplit('/', workingDir, resource);
+
+ auto proposeRequest = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ NKikimrTxUserProxy::TEvProposeTransaction& record = proposeRequest->Record;
NKikimrSchemeOp::TModifyScheme* modifyScheme = record.MutableTransaction()->MutableModifyScheme();
modifyScheme->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpModifyACL);
- modifyScheme->SetWorkingDir(TString(workingDir));
- modifyScheme->MutableModifyACL()->SetName(TString(resource));
-
- modifyScheme->MutableModifyACL()->SetDiffACL(ACLDiff_.SerializeAsString());
+ modifyScheme->SetWorkingDir(TString(workingDir));
+ modifyScheme->MutableModifyACL()->SetName(TString(resource));
+
+ modifyScheme->MutableModifyACL()->SetDiffACL(ACLDiff_.SerializeAsString());
Send(MakeTxProxyID(), proposeRequest.Release());
- }
-
+ }
+
STATEFN(StateFunc) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TEvWakeup, HandleWakeup);
hFunc(TEvTxUserProxy::TEvProposeTransactionStatus, HandleProposeTransactionStatus);
- }
- }
-
+ }
+ }
+
void LogModifyACLRequestResultSafe(bool success) const {
- const TString user = (UserToken_ && UserToken_->GetUserSID()) ? UserToken_->GetUserSID() : "Someone";
-
- TModifyPermissionsRequest copy;
+ const TString user = (UserToken_ && UserToken_->GetUserSID()) ? UserToken_->GetUserSID() : "Someone";
+
+ TModifyPermissionsRequest copy;
copy.CopyFrom(Request());
- copy.ClearCredentials();
- copy.ClearResource();
-
- if (success) {
+ copy.ClearCredentials();
+ copy.ClearResource();
+
+ if (success) {
RLOG_SQS_WARN(user << " modified ACL for " << Resource_ << " with request " << copy);
- } else {
+ } else {
RLOG_SQS_ERROR(user << " failed to modify ACL for " << Resource_ << " with request " << copy);
- }
- }
-
+ }
+ }
+
void HandleProposeTransactionStatus(TEvTxUserProxy::TEvProposeTransactionStatus::TPtr& ev) {
- const TEvTxUserProxy::TEvProposeTransactionStatus* msg = ev->Get();
- const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(msg->Record.GetStatus());
- switch (status) {
- case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete:
- case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress: {
+ const TEvTxUserProxy::TEvProposeTransactionStatus* msg = ev->Get();
+ const auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(msg->Record.GetStatus());
+ switch (status) {
+ case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete:
+ case TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecInProgress: {
LogModifyACLRequestResultSafe(true);
- break;
- }
-
- case TEvTxUserProxy::TResultStatus::ResolveError:
- case TEvTxUserProxy::TResultStatus::AccessDenied:
- case TEvTxUserProxy::TResultStatus::ExecError:
- default: {
+ break;
+ }
+
+ case TEvTxUserProxy::TResultStatus::ResolveError:
+ case TEvTxUserProxy::TResultStatus::AccessDenied:
+ case TEvTxUserProxy::TResultStatus::ExecError:
+ default: {
LogModifyACLRequestResultSafe(false);
- MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
- break;
- }
- }
+ MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
+ break;
+ }
+ }
SendReplyAndDie();
- }
-
+ }
+
const TModifyPermissionsRequest& Request() const {
return SourceSqsRequest_.GetModifyPermissions();
}
-private:
- NACLib::TDiffACL ACLDiff_;
- TString Resource_;
-};
-
+private:
+ NACLib::TDiffACL ACLDiff_;
+ TString Resource_;
+};
+
IActor* CreateModifyPermissionsActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) {
return new TModifyPermissionsActor(sourceSqsRequest, std::move(cb));
-}
-
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/params.h b/ydb/core/ymq/actor/params.h
index cc23308af1..abe1154c8d 100644
--- a/ydb/core/ymq/actor/params.h
+++ b/ydb/core/ymq/actor/params.h
@@ -47,12 +47,12 @@ public:
return *this;
}
- TParameters& Uint32(const TString& name, ui32 value) {
- DataType(name, NScheme::NTypeIds::Uint32);
- Params_->MutableValue()->AddStruct()->SetUint32(value);
- return *this;
- }
-
+ TParameters& Uint32(const TString& name, ui32 value) {
+ DataType(name, NScheme::NTypeIds::Uint32);
+ Params_->MutableValue()->AddStruct()->SetUint32(value);
+ return *this;
+ }
+
TParameters& OptionalBool(const TString& name, const TMaybe<bool>& value) {
OptionalDataType(name, NScheme::NTypeIds::Bool);
if (value) {
@@ -79,17 +79,17 @@ public:
return *this;
}
- TParameters& OptionalUtf8(const TString& name, const TMaybe<TString>& value) {
- OptionalDataType(name, NScheme::NTypeIds::Utf8);
- if (value) {
- Params_->MutableValue()->AddStruct()->MutableOptional()->SetText(*value);
- } else {
- Params_->MutableValue()->AddStruct();
- }
-
- return *this;
- }
-
+ TParameters& OptionalUtf8(const TString& name, const TMaybe<TString>& value) {
+ OptionalDataType(name, NScheme::NTypeIds::Utf8);
+ if (value) {
+ Params_->MutableValue()->AddStruct()->MutableOptional()->SetText(*value);
+ } else {
+ Params_->MutableValue()->AddStruct();
+ }
+
+ return *this;
+ }
+
private:
void DataType(const TString& name, NScheme::TTypeId typeId) {
auto* member = Params_->MutableType()->MutableStruct()->AddMember();
diff --git a/ydb/core/ymq/actor/proxy_actor.cpp b/ydb/core/ymq/actor/proxy_actor.cpp
index cd2b9c0b8b..98d85210a5 100644
--- a/ydb/core/ymq/actor/proxy_actor.cpp
+++ b/ydb/core/ymq/actor/proxy_actor.cpp
@@ -1,4 +1,4 @@
-#include "cfg.h"
+#include "cfg.h"
#include "error.h"
#include "proxy_actor.h"
@@ -10,7 +10,7 @@
#include <util/string/builder.h>
#include <util/system/defaults.h>
-
+
namespace NKikimr::NSQS {
void TProxyActor::Bootstrap() {
@@ -21,20 +21,20 @@ void TProxyActor::Bootstrap() {
const auto& cfg = Cfg();
if (cfg.GetYandexCloudMode()) {
- TString securityToken;
-#define SQS_REQUEST_CASE(action) \
- const auto& request = Request_.Y_CAT(Get, action)(); \
- securityToken = ExtractSecurityToken<typename std::remove_reference<decltype(request)>::type, TCredentials>(request);
- SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false));
-#undef SQS_REQUEST_CASE
- TStringBuf tokenBuf(securityToken);
- UserName_ = TString(tokenBuf.NextTok(':'));
- FolderId_ = TString(tokenBuf.NextTok(':'));
-
- // TODO: handle empty cloud id better
+ TString securityToken;
+#define SQS_REQUEST_CASE(action) \
+ const auto& request = Request_.Y_CAT(Get, action)(); \
+ securityToken = ExtractSecurityToken<typename std::remove_reference<decltype(request)>::type, TCredentials>(request);
+ SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false));
+#undef SQS_REQUEST_CASE
+ TStringBuf tokenBuf(securityToken);
+ UserName_ = TString(tokenBuf.NextTok(':'));
+ FolderId_ = TString(tokenBuf.NextTok(':'));
+
+ // TODO: handle empty cloud id better
RLOG_SQS_DEBUG("Proxy actor: used " << UserName_ << " as an account name and " << QueueName_ << " as a queue name");
- }
-
+ }
+
if (!UserName_ || !QueueName_) {
RLOG_SQS_WARN("Validation error: No " << (!UserName_ ? "user name" : "queue name") << " in proxy actor");
SendErrorAndDie(NErrors::INVALID_PARAMETER_VALUE, "Both account and queue name should be specified.");
@@ -107,20 +107,20 @@ void TProxyActor::SendErrorAndDie(const TErrorClass& error, const TString& messa
detailedCounters->APIStatuses.AddError(error.ErrorCode);
}
NKikimrClient::TSqsResponse response;
-#define SQS_REQUEST_CASE(action) \
+#define SQS_REQUEST_CASE(action) \
MakeError(response.Y_CAT(Mutable, action)(), error, message); \
response.Y_CAT(Mutable, action)()->SetRequestId(RequestId_);
- SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false));
+ SQS_SWITCH_REQUEST(Request_, Y_VERIFY(false));
+
+#undef SQS_REQUEST_CASE
-#undef SQS_REQUEST_CASE
-
if (Cfg().GetYandexCloudMode()) {
- response.SetFolderId(FolderId_);
- response.SetIsFifo(false);
- response.SetResourceId(QueueName_);
- }
-
+ response.SetFolderId(FolderId_);
+ response.SetIsFifo(false);
+ response.SetResourceId(QueueName_);
+ }
+
SendReplyAndDie(response);
}
@@ -136,11 +136,11 @@ void TProxyActor::HandleResponse(TSqsEvents::TEvProxySqsResponse::TPtr& ev) {
void TProxyActor::HandleWakeup(TEvWakeup::TPtr&) {
TString actionName;
-#define SQS_REQUEST_CASE(action) actionName = Y_STRINGIZE(action);
+#define SQS_REQUEST_CASE(action) actionName = Y_STRINGIZE(action);
- SQS_SWITCH_REQUEST(Request_, break;);
+ SQS_SWITCH_REQUEST(Request_, break;);
-#undef SQS_REQUEST_CASE
+#undef SQS_REQUEST_CASE
RLOG_SQS_ERROR("Proxy request timeout. User [" << UserName_ << "] Queue [" << QueueName_ << "] Action [" << actionName << "]");
@@ -179,11 +179,11 @@ const TErrorClass& TProxyActor::GetErrorClass(TSqsEvents::TEvProxySqsResponse::E
}
bool TProxyActor::NeedCreateProxyActor(const NKikimrClient::TSqsRequest& req) {
-#define SQS_REQUEST_CASE(action) return true;
+#define SQS_REQUEST_CASE(action) return true;
- SQS_SWITCH_REQUEST(req, return false)
+ SQS_SWITCH_REQUEST(req, return false)
-#undef SQS_REQUEST_CASE
+#undef SQS_REQUEST_CASE
}
bool TProxyActor::NeedCreateProxyActor(EAction action) {
@@ -191,14 +191,14 @@ bool TProxyActor::NeedCreateProxyActor(EAction action) {
}
void TProxyActor::RetrieveUserAndQueueParameters() {
-// User name might be changed later in bootstrap for cloud mode
-#define SQS_REQUEST_CASE(action) \
+// User name might be changed later in bootstrap for cloud mode
+#define SQS_REQUEST_CASE(action) \
UserName_ = Request_.Y_CAT(Get, action)().GetAuth().GetUserName(); \
QueueName_ = Request_.Y_CAT(Get, action)().GetQueueName(); \
- SQS_SWITCH_REQUEST(Request_, throw TSQSException(NErrors::INVALID_ACTION) << "Incorrect request type")
+ SQS_SWITCH_REQUEST(Request_, throw TSQSException(NErrors::INVALID_ACTION) << "Incorrect request type")
- #undef SQS_REQUEST_CASE
+ #undef SQS_REQUEST_CASE
}
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/proxy_actor.h b/ydb/core/ymq/actor/proxy_actor.h
index 123185c3ea..b553716bf0 100644
--- a/ydb/core/ymq/actor/proxy_actor.h
+++ b/ydb/core/ymq/actor/proxy_actor.h
@@ -14,24 +14,24 @@
namespace NKikimr::NSQS {
-#define SQS_REQUEST_CASE_WRAP(action) \
- case NKikimrClient::TSqsRequest::Y_CAT(k, action): { \
- SQS_REQUEST_CASE(action) \
- break; \
- }
-
-// DO NOT proxy account creation or queue listing
-
-#define SQS_SWITCH_REQUEST_CUSTOM(request, enumerate, default_case) \
- switch ((request).GetRequestCase()) { \
- enumerate(SQS_REQUEST_CASE_WRAP) \
- default: \
- default_case; \
- }
-
-#define SQS_SWITCH_REQUEST(request, default_case) \
- SQS_SWITCH_REQUEST_CUSTOM(request, ENUMERATE_PROXY_ACTIONS, default_case)
-
+#define SQS_REQUEST_CASE_WRAP(action) \
+ case NKikimrClient::TSqsRequest::Y_CAT(k, action): { \
+ SQS_REQUEST_CASE(action) \
+ break; \
+ }
+
+// DO NOT proxy account creation or queue listing
+
+#define SQS_SWITCH_REQUEST_CUSTOM(request, enumerate, default_case) \
+ switch ((request).GetRequestCase()) { \
+ enumerate(SQS_REQUEST_CASE_WRAP) \
+ default: \
+ default_case; \
+ }
+
+#define SQS_SWITCH_REQUEST(request, default_case) \
+ SQS_SWITCH_REQUEST_CUSTOM(request, ENUMERATE_PROXY_ACTIONS, default_case)
+
class TProxyActor
: public TActorBootstrapped<TProxyActor>
{
@@ -81,7 +81,7 @@ private:
NKikimrClient::TSqsRequest Request_;
TString QueueName_;
TString UserName_;
- TString FolderId_;
+ TString FolderId_;
THolder<IReplyCallback> Cb_;
bool ErrorResponse_ = false;
TInstant StartTs_;
diff --git a/ydb/core/ymq/actor/purge_queue.cpp b/ydb/core/ymq/actor/purge_queue.cpp
index 20ed641fde..72830bdd4a 100644
--- a/ydb/core/ymq/actor/purge_queue.cpp
+++ b/ydb/core/ymq/actor/purge_queue.cpp
@@ -21,7 +21,7 @@ public:
{
CopyAccountName(Request());
Response_.MutablePurgeQueue()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
@@ -121,11 +121,11 @@ private:
const auto& entry = Request().GetEntries(i);
auto& req = *ret[i].MutablePurgeQueue();
req.MutableAuth()->SetUserName(UserName_);
-
+
if (Request().HasCredentials()) {
*req.MutableCredentials() = Request().GetCredentials();
- }
-
+ }
+
req.SetQueueName(entry.GetQueueName());
req.SetId(entry.GetId());
}
diff --git a/ydb/core/ymq/actor/queue_leader.cpp b/ydb/core/ymq/actor/queue_leader.cpp
index 2840b6ef09..2c6e81d827 100644
--- a/ydb/core/ymq/actor/queue_leader.cpp
+++ b/ydb/core/ymq/actor/queue_leader.cpp
@@ -41,7 +41,7 @@ const TString INFLY_INVALIDATION_REASON_DELETED = "MessageDeleted";
TQueueLeader::TQueueLeader(TString userName, TString queueName, TString folderId, TString rootUrl, TIntrusivePtr<TQueueCounters> counters, TIntrusivePtr<TUserCounters> userCounters, const TActorId& schemeCache, const TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions>& quoterResourcesForUser)
: UserName_(std::move(userName))
, QueueName_(std::move(queueName))
- , FolderId_(std::move(folderId))
+ , FolderId_(std::move(folderId))
, RootUrl_(std::move(rootUrl))
, SchemeCache_(schemeCache)
, Counters_(std::move(counters))
@@ -260,13 +260,13 @@ void TQueueLeader::HandleExecuteWhileWorking(TSqsEvents::TEvExecute::TPtr& ev) {
void TQueueLeader::Prepare(TSqsEvents::TEvExecute::TPtr& ev) {
const TSqsEvents::TEvExecute& req = *ev->Get();
RLOG_SQS_REQ_DEBUG(req.RequestId, "Preparing query(idx=" << req.QueryIdx << ")");
-
+
TExecutorBuilder(SelfId(), req.RequestId)
.User(UserName_)
.Queue(QueueName_)
.Shard(req.Shard)
- .QueueVersion(QueueVersion_)
- .Fifo(IsFifoQueue_)
+ .QueueVersion(QueueVersion_)
+ .Fifo(IsFifoQueue_)
.Mode(NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE)
.QueryId(req.QueryIdx)
.RetryOnTimeout(req.RetryOnTimeout)
@@ -337,8 +337,8 @@ void TQueueLeader::ExecuteRequest(TSqsEvents::TEvExecute::TPtr& ev, const TStrin
.User(UserName_)
.Queue(QueueName_)
.Shard(req.Shard)
- .QueueVersion(QueueVersion_)
- .Fifo(IsFifoQueue_)
+ .QueueVersion(QueueVersion_)
+ .Fifo(IsFifoQueue_)
.QueryId(req.QueryIdx)
.Bin(compiled)
.RetryOnTimeout(req.RetryOnTimeout)
@@ -556,7 +556,7 @@ void TQueueLeader::OnFifoGroupLocked(const TString& requestId, const TSqsEvents:
auto& msg = reqInfo.LockedFifoMessages.back();
msg.RandomId = offsets[i]["RandomId"];
msg.Offset = offsets[i]["Head"];
- msg.GroupId = offsets[i]["GroupId"];
+ msg.GroupId = offsets[i]["GroupId"];
}
if (truncated) {
@@ -579,15 +579,15 @@ void TQueueLeader::OnFifoGroupLocked(const TString& requestId, const TSqsEvents:
}
void TQueueLeader::ReadFifoMessages(TReceiveMessageBatchRequestProcessing& reqInfo) {
- ui32 maxReceiveCount = 0; // not set
+ ui32 maxReceiveCount = 0; // not set
if (Cfg().GetEnableDeadLetterQueues() && DlqInfo_) {
- const auto& dlqInfo(*DlqInfo_);
- if (dlqInfo.DlqName && dlqInfo.QueueId) {
- // dlq is set and resolved
- maxReceiveCount = dlqInfo.MaxReceiveCount;
- }
- }
-
+ const auto& dlqInfo(*DlqInfo_);
+ if (dlqInfo.DlqName && dlqInfo.QueueId) {
+ // dlq is set and resolved
+ maxReceiveCount = dlqInfo.MaxReceiveCount;
+ }
+ }
+
TExecutorBuilder builder(SelfId(), reqInfo.Event->Get()->RequestId);
builder
.User(UserName_)
@@ -600,46 +600,46 @@ void TQueueLeader::ReadFifoMessages(TReceiveMessageBatchRequestProcessing& reqIn
NClient::TWriteValue params = builder.ParamsValue();
params["NOW"] = ui64(TActivationContext::Now().MilliSeconds());
- ui64 index = 0;
-
- THashSet<TString> usedGroups; // mitigates extremely rare bug with duplicated groups during locking
+ ui64 index = 0;
+
+ THashSet<TString> usedGroups; // mitigates extremely rare bug with duplicated groups during locking
for (const auto& msg : reqInfo.LockedFifoMessages) {
- if (usedGroups.insert(msg.GroupId).second) {
- auto key = params["KEYS"].AddListItem();
-
- key["RandomId"] = msg.RandomId;
- key["Offset"] = msg.Offset;
-
- if (maxReceiveCount) {
- key["GroupId"].Bytes(msg.GroupId);
- key["Index"] = index++;
- }
- }
- }
-
- if (maxReceiveCount) {
- // perform heavy read and move transaction (DLQ)
- Y_VERIFY(DlqInfo_);
-
+ if (usedGroups.insert(msg.GroupId).second) {
+ auto key = params["KEYS"].AddListItem();
+
+ key["RandomId"] = msg.RandomId;
+ key["Offset"] = msg.Offset;
+
+ if (maxReceiveCount) {
+ key["GroupId"].Bytes(msg.GroupId);
+ key["Index"] = index++;
+ }
+ }
+ }
+
+ if (maxReceiveCount) {
+ // perform heavy read and move transaction (DLQ)
+ Y_VERIFY(DlqInfo_);
+
const TQueuePath currentQueuePath = { Cfg().GetRoot(), UserName_, QueueName_, QueueVersion_ };
const TQueuePath deadLetterQueuePath = { Cfg().GetRoot(), UserName_, DlqInfo_->QueueId, DlqInfo_->QueueVersion };
-
- const TString transactionText = Sprintf(GetFifoQueryById(READ_OR_REDRIVE_MESSAGE_ID),
- currentQueuePath.GetVersionedQueuePath().c_str(),
- 0,
- deadLetterQueuePath.GetVersionedQueuePath().c_str());
- builder
- .Text(transactionText)
- .Params()
- .Uint32("MAX_RECEIVE_COUNT", maxReceiveCount)
- .Uint64("RANDOM_ID", RandomNumber<ui64>());
- } else {
- builder
- .QueryId(READ_MESSAGE_ID);
- }
-
- const bool usedDLQ = maxReceiveCount > 0;
-
+
+ const TString transactionText = Sprintf(GetFifoQueryById(READ_OR_REDRIVE_MESSAGE_ID),
+ currentQueuePath.GetVersionedQueuePath().c_str(),
+ 0,
+ deadLetterQueuePath.GetVersionedQueuePath().c_str());
+ builder
+ .Text(transactionText)
+ .Params()
+ .Uint32("MAX_RECEIVE_COUNT", maxReceiveCount)
+ .Uint64("RANDOM_ID", RandomNumber<ui64>());
+ } else {
+ builder
+ .QueryId(READ_MESSAGE_ID);
+ }
+
+ const bool usedDLQ = maxReceiveCount > 0;
+
builder.OnExecuted([this, requestId = reqInfo.Event->Get()->RequestId, usedDLQ] (const TSqsEvents::TEvExecuted::TRecord& ev) {
OnFifoMessagesRead(requestId, ev, usedDLQ);
});
@@ -657,24 +657,24 @@ void TQueueLeader::OnFifoMessagesRead(const TString& requestId, const TSqsEvents
const TValue value(TValue::Create(ev.GetExecutionEngineEvaluatedResponse()));
const TValue list(value["result"]);
- if (const ui64 movedMessagesCount = value["movedMessagesCount"]) {
- ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount);
+ if (const ui64 movedMessagesCount = value["movedMessagesCount"]) {
+ ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount);
const i64 newMessagesCount = value["newMessagesCount"];
Y_VERIFY(newMessagesCount >= 0);
auto& shardInfo = Shards_[0];
shardInfo.MessagesCount = static_cast<ui64>(newMessagesCount);
- }
-
+ }
+
reqInfo.Answer->Messages.resize(list.Size());
for (size_t i = 0; i < list.Size(); ++i) {
- const TValue& data = list[i]["SourceDataFieldsRead"];
- const TValue& msg = list[i]["SourceMessageFieldsRead"];
+ const TValue& data = list[i]["SourceDataFieldsRead"];
+ const TValue& msg = list[i]["SourceMessageFieldsRead"];
const ui64 receiveTimestamp = msg["FirstReceiveTimestamp"];
auto& msgAnswer = reqInfo.Answer->Messages[i];
msgAnswer.FirstReceiveTimestamp = (receiveTimestamp ? TInstant::MilliSeconds(receiveTimestamp) : reqInfo.LockSendTs);
- msgAnswer.ReceiveCount = ui32(msg["ReceiveCount"]) + 1; // since the query returns old receive count value
+ msgAnswer.ReceiveCount = ui32(msg["ReceiveCount"]) + 1; // since the query returns old receive count value
msgAnswer.MessageId = data["MessageId"];
msgAnswer.MessageDeduplicationId = data["DedupId"];
msgAnswer.MessageGroupId = msg["GroupId"];
@@ -701,15 +701,15 @@ void TQueueLeader::OnFifoMessagesRead(const TString& requestId, const TSqsEvents
}
}
} else {
- const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(ev.GetStatus());
- if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) {
- // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info
- DlqInfo_.Clear();
- reqInfo.Answer->Failed = false;
- reqInfo.Answer->Messages.clear();
- } else {
- reqInfo.Answer->Failed = true;
- }
+ const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(ev.GetStatus());
+ if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) {
+ // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info
+ DlqInfo_.Clear();
+ reqInfo.Answer->Failed = false;
+ reqInfo.Answer->Messages.clear();
+ } else {
+ reqInfo.Answer->Failed = true;
+ }
}
Reply(reqInfo);
@@ -756,17 +756,17 @@ void TQueueLeader::OnLoadStdMessageResult(const TString& requestId, const ui64 o
bool deleted = true;
bool deadlineChanged = true;
const bool exists = (*messageRecord)["Exists"];
- const auto wasDeadLetterValue = (*messageRecord)["IsDeadLetter"];
- const bool wasDeadLetter = wasDeadLetterValue.HaveValue() ? bool(wasDeadLetterValue) : false;
-
+ const auto wasDeadLetterValue = (*messageRecord)["IsDeadLetter"];
+ const bool wasDeadLetter = wasDeadLetterValue.HaveValue() ? bool(wasDeadLetterValue) : false;
+
const bool valid = (*messageRecord)["Valid"];
- if (exists && !wasDeadLetter) {
+ if (exists && !wasDeadLetter) {
const ui64 visibilityDeadlineMs = (*messageRecord)["VisibilityDeadline"];
- const ui32 receiveCount = (*messageRecord)["ReceiveCount"];
+ const ui32 receiveCount = (*messageRecord)["ReceiveCount"];
const TInstant visibilityDeadline = TInstant::MilliSeconds(visibilityDeadlineMs);
- // Update actual visibility deadline and receive count even if this message won't be given to user in this request.
+ // Update actual visibility deadline and receive count even if this message won't be given to user in this request.
// It prevents such synchronization errors later.
- reqInfo.ReceiveCandidates.SetVisibilityDeadlineAndReceiveCount(offset, visibilityDeadline, receiveCount);
+ reqInfo.ReceiveCandidates.SetVisibilityDeadlineAndReceiveCount(offset, visibilityDeadline, receiveCount);
if (valid && reqInfo.ReceiveCandidates.Has(offset)) { // there may be concurrent successful delete message request (purge)
reqInfo.Answer->Messages.emplace_back();
@@ -807,8 +807,8 @@ void TQueueLeader::OnLoadStdMessageResult(const TString& requestId, const ui64 o
deleted = false; // Success, not invalidated
} else {
RLOG_SQS_REQ_WARN(requestId, "Attempted to receive message that was deleted. Shard: " << reqInfo.GetCurrentShard() << ". Offset: " << offset);
- deleted = true;
- }
+ deleted = true;
+ }
} // else there was concurrent delete (purge) by this leader, => OK
}
const bool invalidated = deleted || deadlineChanged;
@@ -819,8 +819,8 @@ void TQueueLeader::OnLoadStdMessageResult(const TString& requestId, const ui64 o
MarkInflyReloading(reqInfo.GetCurrentShard(), 1, reason);
}
} else {
- reqInfo.LoadError = !ignoreMessageLoadingErrors;
- // there may be other successful loads
+ reqInfo.LoadError = !ignoreMessageLoadingErrors;
+ // there may be other successful loads
}
if (reqInfo.LoadAnswersLeft == 0) {
@@ -838,21 +838,21 @@ void TQueueLeader::OnLoadStdMessagesBatchExecuted(ui64 shard, ui64 batchId, cons
Y_VERIFY(batchIt != batchingState.BatchesExecuting.end());
auto batch = batchIt->second;
auto status = TEvTxUserProxy::TEvProposeTransactionStatus::EStatus(reply.GetStatus());
- bool ignoreMessageLoadingErrors = false;
+ bool ignoreMessageLoadingErrors = false;
if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
using NKikimr::NClient::TValue;
const TValue value(TValue::Create(reply.GetExecutionEngineEvaluatedResponse()));
const TValue list(value["result"]);
Y_VERIFY(list.Size() == batch->Size());
- if (const ui64 movedMessagesCount = value["movedMessagesCount"]) {
- ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount);
+ if (const ui64 movedMessagesCount = value["movedMessagesCount"]) {
+ ADD_COUNTER(Counters_, MessagesMovedToDLQ, movedMessagesCount);
const i64 newMessagesCount = value["newMessagesCount"];
Y_VERIFY(newMessagesCount >= 0);
shardInfo.MessagesCount = static_cast<ui64>(newMessagesCount);
- }
-
+ }
+
THashMap<ui64, const TLoadBatchEntry*> offset2entry;
offset2entry.reserve(batch->Entries.size());
for (const TLoadBatchEntry& entry : batch->Entries) {
@@ -867,13 +867,13 @@ void TQueueLeader::OnLoadStdMessagesBatchExecuted(ui64 shard, ui64 batchId, cons
OnLoadStdMessageResult(entry->second->RequestId, offset, reply, &msg, ignoreMessageLoadingErrors);
}
} else {
- const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(reply.GetStatus());
- if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) {
- // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info
- DlqInfo_.Clear();
- ignoreMessageLoadingErrors = true;
- }
-
+ const auto errStatus = NKikimr::NTxProxy::TResultStatus::EStatus(reply.GetStatus());
+ if (usedDLQ && !NTxProxy::TResultStatus::IsSoftErrorWithoutSideEffects(errStatus)) {
+ // it's possible that DLQ was removed, hence it'd be wise to refresh corresponding info
+ DlqInfo_.Clear();
+ ignoreMessageLoadingErrors = true;
+ }
+
const TString* prevRequestId = nullptr;
for (size_t i = 0; i < batch->Size(); ++i) {
const TLoadBatchEntry& entry = batch->Entries[i];
@@ -1269,10 +1269,10 @@ void TQueueLeader::OnQueueConfiguration(const TSqsEvents::TEvExecuted::TRecord&
ShardsCount_ = data["Shards"];
PartitionsCount_ = data["Partitions"];
QueueId_ = data["QueueId"];
- if (data["Version"].HaveValue()) {
- QueueVersion_ = ui64(data["Version"]);
- }
- IsFifoQueue_ = bool(data["FifoQueue"]);
+ if (data["Version"].HaveValue()) {
+ QueueVersion_ = ui64(data["Version"]);
+ }
+ IsFifoQueue_ = bool(data["FifoQueue"]);
Shards_.resize(ShardsCount_);
const auto& cfg = Cfg();
if (IsFifoQueue_) {
@@ -1362,8 +1362,8 @@ void TQueueLeader::AskQueueAttributes() {
.Queue(QueueName_)
.QueueLeader(SelfId())
.QueryId(INTERNAL_GET_QUEUE_ATTRIBUTES_ID)
- .QueueVersion(QueueVersion_)
- .Fifo(IsFifoQueue_)
+ .QueueVersion(QueueVersion_)
+ .Fifo(IsFifoQueue_)
.RetryOnTimeout()
.OnExecuted([this](const TSqsEvents::TEvExecuted::TRecord& ev) { OnQueueAttributes(ev); })
.Counters(Counters_)
@@ -1392,22 +1392,22 @@ void TQueueLeader::OnQueueAttributes(const TSqsEvents::TEvExecuted::TRecord& ev)
Counters_->ShowDetailedCounters(TInstant::MilliSeconds(ms));
}
- // update dead letter queue info
- const auto& dlqNameVal(attrs["DlqName"]);
- const auto& maxReceiveCountVal(attrs["MaxReceiveCount"]);
- if (dlqNameVal.HaveValue() && maxReceiveCountVal.HaveValue()) {
- TTargetDlqInfo info;
- info.DlqName = TString(dlqNameVal);
- info.MaxReceiveCount = ui64(maxReceiveCountVal);
- if (info.DlqName && info.MaxReceiveCount) {
- DlqInfo_ = info;
- // now we have to discover queue id and version
+ // update dead letter queue info
+ const auto& dlqNameVal(attrs["DlqName"]);
+ const auto& maxReceiveCountVal(attrs["MaxReceiveCount"]);
+ if (dlqNameVal.HaveValue() && maxReceiveCountVal.HaveValue()) {
+ TTargetDlqInfo info;
+ info.DlqName = TString(dlqNameVal);
+ info.MaxReceiveCount = ui64(maxReceiveCountVal);
+ if (info.DlqName && info.MaxReceiveCount) {
+ DlqInfo_ = info;
+ // now we have to discover queue id and version
Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvGetQueueId("DLQ", UserName_, info.DlqName, FolderId_));
- } else {
- DlqInfo_.Clear();
- }
- }
-
+ } else {
+ DlqInfo_.Clear();
+ }
+ }
+
QueueAttributes_ = attributes;
AttributesUpdateTime_ = TActivationContext::Now();
for (auto& req : GetConfigurationRequests_) {
@@ -1423,26 +1423,26 @@ void TQueueLeader::OnQueueAttributes(const TSqsEvents::TEvExecuted::TRecord& ev)
}
void TQueueLeader::HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev) {
- if (!DlqInfo_) {
- return;
- }
-
- if (ev->Get()->Failed) {
+ if (!DlqInfo_) {
+ return;
+ }
+
+ if (ev->Get()->Failed) {
LOG_SQS_DEBUG("Dlq discovering failed");
- } else {
- if (ev->Get()->Exists) {
- DlqInfo_->QueueId = ev->Get()->QueueId;
- DlqInfo_->QueueVersion = ev->Get()->Version;
- DlqInfo_->ShardsCount = ev->Get()->ShardsCount;
-
+ } else {
+ if (ev->Get()->Exists) {
+ DlqInfo_->QueueId = ev->Get()->QueueId;
+ DlqInfo_->QueueVersion = ev->Get()->Version;
+ DlqInfo_->ShardsCount = ev->Get()->ShardsCount;
+
LOG_SQS_DEBUG("Discovered DLQ: name: " << DlqInfo_->DlqName << ", maxReceiveCount: " << DlqInfo_->MaxReceiveCount << ", queueId: " << DlqInfo_->QueueId << ", version: " << DlqInfo_->QueueVersion << ", shards count: " << DlqInfo_->ShardsCount);
- return;
- }
- }
-
- DlqInfo_.Clear(); // something is off
-}
-
+ return;
+ }
+ }
+
+ DlqInfo_.Clear(); // something is off
+}
+
void TQueueLeader::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
ev->Get()->Call();
}
@@ -1457,11 +1457,11 @@ void TQueueLeader::StartGatheringMetrics() {
if (IsDlqQueue_) {
LOG_SQS_INFO("Stopped periodic message counting for queue " << TLogQueueName(UserName_, QueueName_)
<< ". Latest dlq notification was at " << LatestDlqNotificationTs_);
- }
-
+ }
+
IsDlqQueue_ = false;
- }
-
+ }
+
for (ui64 shard = 0; shard < ShardsCount_; ++shard) {
if (IsFifoQueue_ || IsDlqQueue_) {
RequestMessagesCountMetrics(shard);
@@ -1746,7 +1746,7 @@ void TQueueLeader::OnInflyLoaded(ui64 shard, const TSqsEvents::TEvExecuted::TRec
DelayStatistics_.AddDelayedMessage(delayDeadline, now);
}
const ui64 offset = message["Offset"];
- const ui32 receiveCount = message["ReceiveCount"];
+ const ui32 receiveCount = message["ReceiveCount"];
const TInstant maxVisibilityDeadline = TInstant::MilliSeconds(Max(visibilityDeadlineMs, delayDeadlineMs));
LOG_SQS_TRACE("Adding message to infly struct for shard " << TLogQueueName(UserName_, QueueName_, shard) << ": { Offset: " << offset << ", VisibilityDeadline: " << maxVisibilityDeadline << ", ReceiveCount: " << receiveCount << " }");
shardInfo.Infly->Add(MakeHolder<TInflyMessage>(offset, message["RandomId"], maxVisibilityDeadline, receiveCount));
@@ -1866,7 +1866,7 @@ void TQueueLeader::OnAddedMessagesToInfly(ui64 shard, const TSqsEvents::TEvExecu
const ui64 delayDeadlineMs = delayDeadlineValue.HaveValue() ? ui64(delayDeadlineValue) : 0;
const TInstant delayDeadline = TInstant::MilliSeconds(delayDeadlineMs);
const ui64 offset = message["Offset"];
- const ui32 receiveCount = 0; // as in transaction
+ const ui32 receiveCount = 0; // as in transaction
LOG_SQS_TRACE("Adding message to infly struct for shard " << TLogQueueName(UserName_, QueueName_, shard) << ": { Offset: " << offset << ", DelayDeadline: " << delayDeadline << ", ReceiveCount: " << receiveCount << " }");
shardInfo.Infly->Add(MakeHolder<TInflyMessage>(offset, message["RandomId"], delayDeadline, receiveCount));
}
@@ -2118,17 +2118,17 @@ void TQueueLeader::HandleGetRuntimeQueueAttributesWhileWorking(TSqsEvents::TEvGe
void TQueueLeader::HandleDeadLetterQueueNotification(TSqsEvents::TEvDeadLetterQueueNotification::TPtr&) {
LatestDlqNotificationTs_ = TActivationContext::Now();
-
+
if (!IsFifoQueue_ && !IsDlqQueue_) {
- // we need to start the process only once
+ // we need to start the process only once
IsDlqQueue_ = true;
LOG_SQS_INFO("Started periodic message counting for queue " << TLogQueueName(UserName_, QueueName_)
<< ". Latest dlq notification was at " << LatestDlqNotificationTs_);
-
+
StartGatheringMetrics();
- }
-}
-
+ }
+}
+
void TQueueLeader::ProcessGetRuntimeQueueAttributes(TGetRuntimeQueueAttributesRequestProcessing& reqInfo) {
if (reqInfo.ShardProcessFlags.empty()) {
Y_VERIFY(ShardsCount_ > 0);
@@ -2521,7 +2521,7 @@ void TQueueLeader::TLoadBatchingState::AddRequest(TReceiveMessageBatchRequestPro
void TQueueLeader::TLoadBatch::Execute(TQueueLeader* leader) {
RLOG_SQS_DEBUG(TLogQueueName(leader->UserName_, leader->QueueName_, Shard) << " Executing load batch. BatchId: " << BatchId << ". Size: " << Size());
-
+
TExecutorBuilder builder(SelfId(), RequestId_);
const auto now = TActivationContext::Now();
builder
@@ -2538,63 +2538,63 @@ void TQueueLeader::TLoadBatch::Execute(TQueueLeader* leader) {
.Uint64("READ_ID", RandomNumber<ui64>())
.Uint64("SHARD", Shard);
- ui32 maxReceiveCount = 0; // not set
+ ui32 maxReceiveCount = 0; // not set
if (Cfg().GetEnableDeadLetterQueues() && leader->DlqInfo_) {
const auto& dlqInfo(*leader->DlqInfo_);
- if (dlqInfo.DlqName && dlqInfo.QueueId) {
- // dlq is set and resolved
- maxReceiveCount = dlqInfo.MaxReceiveCount;
- }
- }
-
+ if (dlqInfo.DlqName && dlqInfo.QueueId) {
+ // dlq is set and resolved
+ maxReceiveCount = dlqInfo.MaxReceiveCount;
+ }
+ }
+
NClient::TWriteValue params = builder.ParamsValue();
const TString* prevRequestId = nullptr;
- size_t deadLettersCounter = 0;
- THashSet<ui64> offsets; // check for duplicates
+ size_t deadLettersCounter = 0;
+ THashSet<ui64> offsets; // check for duplicates
for (const TLoadBatchEntry& entry : Entries) {
- Y_VERIFY(offsets.insert(entry.Offset).second);
-
+ Y_VERIFY(offsets.insert(entry.Offset).second);
+
auto item = params["KEYS"].AddListItem();
item["RandomId"] = entry.RandomId;
item["Offset"] = entry.Offset;
item["CurrentVisibilityDeadline"] = ui64(entry.CurrentVisibilityDeadline.MilliSeconds());
item["VisibilityDeadline"] = ui64((now + entry.VisibilityTimeout).MilliSeconds());
- if (maxReceiveCount && entry.ReceiveCount >= maxReceiveCount) {
- item["DlqIndex"] = ui64(deadLettersCounter);
- ++deadLettersCounter;
- item["IsDeadLetter"] = true;
- } else {
- item["DlqIndex"] = ui64(0);
- item["IsDeadLetter"] = false;
- }
-
+ if (maxReceiveCount && entry.ReceiveCount >= maxReceiveCount) {
+ item["DlqIndex"] = ui64(deadLettersCounter);
+ ++deadLettersCounter;
+ item["IsDeadLetter"] = true;
+ } else {
+ item["DlqIndex"] = ui64(0);
+ item["IsDeadLetter"] = false;
+ }
+
if (!prevRequestId || *prevRequestId != entry.RequestId) {
prevRequestId = &entry.RequestId;
RLOG_SQS_REQ_DEBUG(entry.RequestId, "Send batch transaction to database. BatchId: " << BatchId);
}
}
- if (deadLettersCounter) {
- // perform heavy read and move transaction (DLQ)
+ if (deadLettersCounter) {
+ // perform heavy read and move transaction (DLQ)
Y_VERIFY(leader->DlqInfo_);
const auto& dlqInfo(*leader->DlqInfo_);
-
+
const TQueuePath currentQueuePath = { Cfg().GetRoot(), leader->UserName_, leader->QueueName_, leader->QueueVersion_ };
const TQueuePath deadLetterQueuePath = { Cfg().GetRoot(), leader->UserName_, dlqInfo.QueueId, dlqInfo.QueueVersion };
-
- const TString transactionText = Sprintf(GetStdQueryById(LOAD_OR_REDRIVE_MESSAGE_ID),
- currentQueuePath.GetVersionedQueuePath().c_str(), Shard,
- deadLetterQueuePath.GetVersionedQueuePath().c_str(), Shard % dlqInfo.ShardsCount); // TODO: shard inserts aren't balanced
-
- builder.Text(transactionText)
- .Params()
- .Uint64("DEAD_LETTERS_COUNT", deadLettersCounter);
- } else {
- // perform simple read transaction
- builder.QueryId(LOAD_MESSAGES_ID);
- }
-
- const bool usedDLQ = deadLettersCounter;
+
+ const TString transactionText = Sprintf(GetStdQueryById(LOAD_OR_REDRIVE_MESSAGE_ID),
+ currentQueuePath.GetVersionedQueuePath().c_str(), Shard,
+ deadLetterQueuePath.GetVersionedQueuePath().c_str(), Shard % dlqInfo.ShardsCount); // TODO: shard inserts aren't balanced
+
+ builder.Text(transactionText)
+ .Params()
+ .Uint64("DEAD_LETTERS_COUNT", deadLettersCounter);
+ } else {
+ // perform simple read transaction
+ builder.QueryId(LOAD_MESSAGES_ID);
+ }
+
+ const bool usedDLQ = deadLettersCounter;
builder.OnExecuted([leader, shard = Shard, batchId = BatchId, usedDLQ] (const TSqsEvents::TEvExecuted::TRecord& ev) {
leader->OnLoadStdMessagesBatchExecuted(shard, batchId, usedDLQ, ev);
});
diff --git a/ydb/core/ymq/actor/queue_leader.h b/ydb/core/ymq/actor/queue_leader.h
index adf6edadef..3504e4ddb1 100644
--- a/ydb/core/ymq/actor/queue_leader.h
+++ b/ydb/core/ymq/actor/queue_leader.h
@@ -153,13 +153,13 @@ private:
// const info
TString UserName_;
TString QueueName_;
- TString FolderId_;
+ TString FolderId_;
TString RootUrl_;
ui64 ShardsCount_ = 0;
ui64 PartitionsCount_ = 0;
- bool IsFifoQueue_ = false;
+ bool IsFifoQueue_ = false;
TString QueueId_;
- ui64 QueueVersion_ = 0;
+ ui64 QueueVersion_ = 0;
TActorId SchemeCache_;
TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> QuoterResources_;
TLocalRateLimiterResource SendMessageQuoterResource_;
@@ -177,20 +177,20 @@ private:
TIntrusivePtr<TUserCounters> UserCounters_;
size_t MetricsQueriesInfly_ = 0;
- // dead letter queue params
- struct TTargetDlqInfo {
- // from attributes
- TString DlqName;
- ui64 MaxReceiveCount = 0;
- // discovered via service
- TString QueueId; // unique name or resource id in cloud
- ui64 QueueVersion = 0;
- ui64 ShardsCount = 0;
- };
- TMaybe<TTargetDlqInfo> DlqInfo_;
+ // dead letter queue params
+ struct TTargetDlqInfo {
+ // from attributes
+ TString DlqName;
+ ui64 MaxReceiveCount = 0;
+ // discovered via service
+ TString QueueId; // unique name or resource id in cloud
+ ui64 QueueVersion = 0;
+ ui64 ShardsCount = 0;
+ };
+ TMaybe<TTargetDlqInfo> DlqInfo_;
bool IsDlqQueue_ = false;
- TInstant LatestDlqNotificationTs_ = TInstant::Zero();
-
+ TInstant LatestDlqNotificationTs_ = TInstant::Zero();
+
// shards
enum class EQueryState {
Empty,
@@ -294,7 +294,7 @@ private:
: RequestId(requestId)
, RandomId(msg->GetRandomId())
, Offset(msg->GetOffset())
- , ReceiveCount(msg->GetReceiveCount())
+ , ReceiveCount(msg->GetReceiveCount())
, CurrentVisibilityDeadline(msg->GetVisibilityDeadline())
, VisibilityTimeout(visibilityTimeout)
{
@@ -303,7 +303,7 @@ private:
TString RequestId;
ui64 RandomId = 0;
ui64 Offset = 0;
- ui32 ReceiveCount = 0;
+ ui32 ReceiveCount = 0;
TInstant CurrentVisibilityDeadline;
TDuration VisibilityTimeout;
};
@@ -455,7 +455,7 @@ private:
struct TLockedFifoMessage {
ui64 RandomId = 0;
ui64 Offset = 0;
- TString GroupId;
+ TString GroupId;
};
std::vector<TLockedFifoMessage> LockedFifoMessages;
};
diff --git a/ydb/core/ymq/actor/queue_schema.cpp b/ydb/core/ymq/actor/queue_schema.cpp
index 1cde9b0fe3..91d3c5d3df 100644
--- a/ydb/core/ymq/actor/queue_schema.cpp
+++ b/ydb/core/ymq/actor/queue_schema.cpp
@@ -1,22 +1,22 @@
#include "cfg.h"
-#include "executor.h"
-#include "log.h"
-#include "params.h"
-#include "queue_schema.h"
-#include "serviceid.h"
-
+#include "executor.h"
+#include "log.h"
+#include "params.h"
+#include "queue_schema.h"
+#include "serviceid.h"
+
#include <ydb/core/ymq/base/limits.h>
#include <ydb/core/ymq/queues/fifo/schema.h>
#include <ydb/core/ymq/queues/std/schema.h>
-
+
#include <util/digest/city.h>
#include <util/generic/utility.h>
#include <util/string/join.h>
-using NKikimr::NClient::TValue;
-
+using NKikimr::NClient::TValue;
+
namespace NKikimr::NSQS {
-
+
constexpr TStringBuf FIFO_TABLES_DIR = ".FIFO";
constexpr TStringBuf STD_TABLES_DIR = ".STD";
@@ -25,145 +25,145 @@ ui64 GetHash(TArgs&&... args) {
return CityHash64(Join("#", args...));
}
-static bool IsGoodStatusCode(ui32 code) {
- switch (NTxProxy::TResultStatus::EStatus(code)) {
- case NTxProxy::TResultStatus::EStatus::ExecComplete:
- case NTxProxy::TResultStatus::EStatus::ExecAlready:
- return true;
- default:
- return false;
- }
-}
-
-TCreateQueueSchemaActorV2::TCreateQueueSchemaActorV2(const TQueuePath& path,
- const TCreateQueueRequest& req,
+static bool IsGoodStatusCode(ui32 code) {
+ switch (NTxProxy::TResultStatus::EStatus(code)) {
+ case NTxProxy::TResultStatus::EStatus::ExecComplete:
+ case NTxProxy::TResultStatus::EStatus::ExecAlready:
+ return true;
+ default:
+ return false;
+ }
+}
+
+TCreateQueueSchemaActorV2::TCreateQueueSchemaActorV2(const TQueuePath& path,
+ const TCreateQueueRequest& req,
const TActorId& sender,
- const TString& requestId,
- const TString& customQueueName,
- const TString& folderId,
- const bool isCloudMode,
- const bool enableQueueAttributesValidation,
+ const TString& requestId,
+ const TString& customQueueName,
+ const TString& folderId,
+ const bool isCloudMode,
+ const bool enableQueueAttributesValidation,
TIntrusivePtr<TUserCounters> userCounters,
TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResources)
- : QueuePath_(path)
- , Request_(req)
- , Sender_(sender)
- , CustomQueueName_(customQueueName)
- , FolderId_(folderId)
- , RequestId_(requestId)
- , GeneratedQueueId_(CreateGuidAsString())
- , QueueCreationTimestamp_(TInstant::Now())
- , IsCloudMode_(isCloudMode)
- , EnableQueueAttributesValidation_(enableQueueAttributesValidation)
+ : QueuePath_(path)
+ , Request_(req)
+ , Sender_(sender)
+ , CustomQueueName_(customQueueName)
+ , FolderId_(folderId)
+ , RequestId_(requestId)
+ , GeneratedQueueId_(CreateGuidAsString())
+ , QueueCreationTimestamp_(TInstant::Now())
+ , IsCloudMode_(isCloudMode)
+ , EnableQueueAttributesValidation_(enableQueueAttributesValidation)
, UserCounters_(std::move(userCounters))
, QuoterResources_(std::move(quoterResources))
-{
- IsFifo_ = AsciiHasSuffixIgnoreCase(IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName, ".fifo");
-
- if (IsFifo_) {
- RequiredShardsCount_ = 1;
- RequiredTables_ = GetFifoTables();
- } else {
- RequiredShardsCount_ = Request_.GetShards();
+{
+ IsFifo_ = AsciiHasSuffixIgnoreCase(IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName, ".fifo");
+
+ if (IsFifo_) {
+ RequiredShardsCount_ = 1;
+ RequiredTables_ = GetFifoTables();
+ } else {
+ RequiredShardsCount_ = Request_.GetShards();
RequiredTables_ = GetStandardTables(RequiredShardsCount_, req.GetPartitions(), req.GetEnableAutosplit(), req.GetSizeToSplit());
- }
-}
-
-TCreateQueueSchemaActorV2::~TCreateQueueSchemaActorV2() = default;
-
+ }
+}
+
+TCreateQueueSchemaActorV2::~TCreateQueueSchemaActorV2() = default;
+
static THolder<TSqsEvents::TEvQueueCreated> MakeErrorResponse(const TErrorClass& errorClass) {
- auto resp = MakeHolder<TSqsEvents::TEvQueueCreated>();
- resp->Success = false;
- resp->State = EQueueState::Active;
+ auto resp = MakeHolder<TSqsEvents::TEvQueueCreated>();
+ resp->Success = false;
+ resp->State = EQueueState::Active;
resp->ErrorClass = &errorClass;
-
- return resp;
-}
-
-template<typename TMaybeAttribute, typename TValueType>
-static void SetDefaultIfMissing(TMaybeAttribute& attribute, const TValueType& defaultVal) {
- if (!attribute) {
- attribute = defaultVal;
- }
-}
-
-static ui64 SecondsToMs(ui64 seconds) {
- return TDuration::Seconds(seconds).MilliSeconds();
-}
-
-void TCreateQueueSchemaActorV2::InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config) {
- // https://docs.aws.amazon.com/en_us/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html
-
- SetDefaultIfMissing(ValidatedAttributes_.DelaySeconds, 0);
- SetDefaultIfMissing(ValidatedAttributes_.MaximumMessageSize, 256 * 1024); // 256 KB
- SetDefaultIfMissing(ValidatedAttributes_.MessageRetentionPeriod,
- Max(TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(), TDuration::Days(4).Seconds()));
- SetDefaultIfMissing(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds, 0); // seconds
- SetDefaultIfMissing(ValidatedAttributes_.VisibilityTimeout, 30); // seconds
- SetDefaultIfMissing(ValidatedAttributes_.ContentBasedDeduplication, false); // bool
-
- // RedrivePolicy could be unspecified
-}
-
+
+ return resp;
+}
+
+template<typename TMaybeAttribute, typename TValueType>
+static void SetDefaultIfMissing(TMaybeAttribute& attribute, const TValueType& defaultVal) {
+ if (!attribute) {
+ attribute = defaultVal;
+ }
+}
+
+static ui64 SecondsToMs(ui64 seconds) {
+ return TDuration::Seconds(seconds).MilliSeconds();
+}
+
+void TCreateQueueSchemaActorV2::InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config) {
+ // https://docs.aws.amazon.com/en_us/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html
+
+ SetDefaultIfMissing(ValidatedAttributes_.DelaySeconds, 0);
+ SetDefaultIfMissing(ValidatedAttributes_.MaximumMessageSize, 256 * 1024); // 256 KB
+ SetDefaultIfMissing(ValidatedAttributes_.MessageRetentionPeriod,
+ Max(TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(), TDuration::Days(4).Seconds()));
+ SetDefaultIfMissing(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds, 0); // seconds
+ SetDefaultIfMissing(ValidatedAttributes_.VisibilityTimeout, 30); // seconds
+ SetDefaultIfMissing(ValidatedAttributes_.ContentBasedDeduplication, false); // bool
+
+ // RedrivePolicy could be unspecified
+}
+
void TCreateQueueSchemaActorV2::Bootstrap() {
- Become(&TCreateQueueSchemaActorV2::Preamble);
-
- THashMap<TString, TString> attributes;
- for (const auto& attr : Request_.attributes()) {
- attributes[attr.GetName()] = attr.GetValue();
- }
-
- const bool clampValues = !EnableQueueAttributesValidation_;
+ Become(&TCreateQueueSchemaActorV2::Preamble);
+
+ THashMap<TString, TString> attributes;
+ for (const auto& attr : Request_.attributes()) {
+ attributes[attr.GetName()] = attr.GetValue();
+ }
+
+ const bool clampValues = !EnableQueueAttributesValidation_;
ValidatedAttributes_ = TQueueAttributes::FromAttributesAndConfig(attributes, Cfg(), IsFifo_, clampValues);
-
- if (!ValidatedAttributes_.Validate()) {
+
+ if (!ValidatedAttributes_.Validate()) {
auto resp = MakeErrorResponse(NErrors::VALIDATION_ERROR);
- resp->Error = ValidatedAttributes_.ErrorText;
-
+ resp->Error = ValidatedAttributes_.ErrorText;
+
Send(Sender_, std::move(resp));
PassAway();
- return;
- }
-
- if (ValidatedAttributes_.HasClampedAttributes()) {
+ return;
+ }
+
+ if (ValidatedAttributes_.HasClampedAttributes()) {
RLOG_SQS_WARN("Clamped some queue attribute values for account " << QueuePath_.UserName << " and queue name " << QueuePath_.QueueName);
- }
-
+ }
+
InitMissingQueueAttributes(Cfg());
-
- if (ValidatedAttributes_.RedrivePolicy.TargetQueueName) {
- const TString createdQueueName = IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName;
+
+ if (ValidatedAttributes_.RedrivePolicy.TargetQueueName) {
+ const TString createdQueueName = IsCloudMode_ ? CustomQueueName_ : QueuePath_.QueueName;
auto resp = MakeErrorResponse(NErrors::VALIDATION_ERROR);
- if (ValidatedAttributes_.RedrivePolicy.TargetQueueName->empty()) {
+ if (ValidatedAttributes_.RedrivePolicy.TargetQueueName->empty()) {
resp->Error = "Empty target dead letter queue name.";
- } else if (*ValidatedAttributes_.RedrivePolicy.TargetQueueName == createdQueueName) {
- resp->Error = "Using the queue itself as a dead letter queue is not allowed.";
- } else {
+ } else if (*ValidatedAttributes_.RedrivePolicy.TargetQueueName == createdQueueName) {
+ resp->Error = "Using the queue itself as a dead letter queue is not allowed.";
+ } else {
Send(MakeSqsServiceID(SelfId().NodeId()),
- new TSqsEvents::TEvGetQueueId(RequestId_, QueuePath_.UserName,
- *ValidatedAttributes_.RedrivePolicy.TargetQueueName, FolderId_));
- return;
- }
-
+ new TSqsEvents::TEvGetQueueId(RequestId_, QueuePath_.UserName,
+ *ValidatedAttributes_.RedrivePolicy.TargetQueueName, FolderId_));
+ return;
+ }
+
Send(Sender_, std::move(resp));
PassAway();
-
- return;
- }
-
+
+ return;
+ }
+
RequestQueueParams();
-}
-
-static const char* const ReadQueueParamsQueryCloud = R"__(
- (
- (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String)))
- (let folderId (Parameter 'FOLDERID (DataType 'Utf8String)))
+}
+
+static const char* const ReadQueueParamsQueryCloud = R"__(
+ (
+ (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String)))
+ (let folderId (Parameter 'FOLDERID (DataType 'Utf8String)))
(let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
-
+
(let queuesTable '%1$s/.Queues)
(let settingsTable '%1$s/.Settings)
-
+
(let maxQueuesCountSettingRow '(
'('Account userName)
'('Name (Utf8String '"MaxQueuesCount"))))
@@ -174,49 +174,49 @@ static const char* const ReadQueueParamsQueryCloud = R"__(
(let queuesRange '(
'('Account userName userName)
- '('QueueName (Utf8String '"") (Void))))
- (let queues
+ '('QueueName (Utf8String '"") (Void))))
+ (let queues
(Member (SelectRange queuesTable queuesRange '('QueueName 'CustomQueueName 'Version 'FolderId) '()) 'List))
- (let overLimit
+ (let overLimit
(LessOrEqual maxQueuesCountSetting (Length queues)))
-
- (let existingQueuesWithSameNameAndFolderId
- (Filter queues (lambda '(item) (block '(
- (return (Coalesce
- (And
- (Equal (Member item 'CustomQueueName) customName)
- (Equal (Member item 'FolderId) folderId))
- (Bool 'false))))))
- )
- )
-
- (let queueExists (NotEqual (Uint64 '0) (Length existingQueuesWithSameNameAndFolderId)))
- (let currentVersion (Coalesce
- (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version)
- (Uint64 '0)
- ))
- (let existingResourceId (Coalesce
- (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName)
- (Utf8String '"")
- ))
-
- (return (AsList
- (SetResult 'exists queueExists)
- (SetResult 'resourceId existingResourceId)
- (SetResult 'overLimit overLimit)
- (SetResult 'version currentVersion)))
- )
-)__";
-
-static const char* const ReadQueueParamsQueryYandex = R"__(
- (
- (let name (Parameter 'NAME (DataType 'Utf8String)))
+
+ (let existingQueuesWithSameNameAndFolderId
+ (Filter queues (lambda '(item) (block '(
+ (return (Coalesce
+ (And
+ (Equal (Member item 'CustomQueueName) customName)
+ (Equal (Member item 'FolderId) folderId))
+ (Bool 'false))))))
+ )
+ )
+
+ (let queueExists (NotEqual (Uint64 '0) (Length existingQueuesWithSameNameAndFolderId)))
+ (let currentVersion (Coalesce
+ (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version)
+ (Uint64 '0)
+ ))
+ (let existingResourceId (Coalesce
+ (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName)
+ (Utf8String '"")
+ ))
+
+ (return (AsList
+ (SetResult 'exists queueExists)
+ (SetResult 'resourceId existingResourceId)
+ (SetResult 'overLimit overLimit)
+ (SetResult 'version currentVersion)))
+ )
+)__";
+
+static const char* const ReadQueueParamsQueryYandex = R"__(
+ (
+ (let name (Parameter 'NAME (DataType 'Utf8String)))
(let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
-
+
(let queuesTable '%1$s/.Queues)
(let settingsTable '%1$s/.Settings)
-
+
(let maxQueuesCountSettingRow '(
'('Account userName)
'('Name (Utf8String '"MaxQueuesCount"))))
@@ -227,131 +227,131 @@ static const char* const ReadQueueParamsQueryYandex = R"__(
(let queuesRange '(
'('Account userName userName)
- '('QueueName (Utf8String '"") (Void))))
- (let queues
+ '('QueueName (Utf8String '"") (Void))))
+ (let queues
(Member (SelectRange queuesTable queuesRange '('QueueState) '()) 'List))
- (let overLimit
+ (let overLimit
(LessOrEqual maxQueuesCountSetting (Length queues)))
-
+
(let queuesRow '(
'('Account userName)
- '('QueueName name)))
+ '('QueueName name)))
(let queuesSelect '(
- 'QueueState
- 'Version))
+ 'QueueState
+ 'Version))
(let queuesRead (SelectRow queuesTable queuesRow queuesSelect))
-
- (let queueExists
- (Coalesce
- (Or
+
+ (let queueExists
+ (Coalesce
+ (Or
(Equal (Uint64 '1) (Member queuesRead 'QueueState))
(Equal (Uint64 '3) (Member queuesRead 'QueueState))
- )
- (Bool 'false)))
-
- (let currentVersion
- (Coalesce
+ )
+ (Bool 'false)))
+
+ (let currentVersion
+ (Coalesce
(Member queuesRead 'Version)
- (Uint64 '0)
- )
- )
-
- (return (AsList
- (SetResult 'exists queueExists)
- (SetResult 'overLimit overLimit)
- (SetResult 'version currentVersion)))
- )
-)__";
-
+ (Uint64 '0)
+ )
+ )
+
+ (return (AsList
+ (SetResult 'exists queueExists)
+ (SetResult 'overLimit overLimit)
+ (SetResult 'version currentVersion)))
+ )
+)__";
+
void TCreateQueueSchemaActorV2::RequestQueueParams() {
- if (IsCloudMode_) {
+ if (IsCloudMode_) {
auto ev = MakeExecuteEvent(Sprintf(ReadQueueParamsQueryCloud, Cfg().GetRoot().c_str()));
- auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
- TParameters(trans->MutableParams()->MutableProto())
- .Utf8("CUSTOMNAME", CustomQueueName_)
+ auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
+ TParameters(trans->MutableParams()->MutableProto())
+ .Utf8("CUSTOMNAME", CustomQueueName_)
.Utf8("FOLDERID", FolderId_)
.Uint64("DEFAULT_MAX_QUEUES_COUNT", Cfg().GetAccountSettingsDefaults().GetMaxQueuesCount())
.Utf8("USER_NAME", QueuePath_.UserName);
-
+
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
- } else {
+ } else {
auto ev = MakeExecuteEvent(Sprintf(ReadQueueParamsQueryYandex, Cfg().GetRoot().c_str()));
- auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
- TParameters(trans->MutableParams()->MutableProto())
+ auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
+ TParameters(trans->MutableParams()->MutableProto())
.Utf8("NAME", QueuePath_.QueueName)
.Uint64("DEFAULT_MAX_QUEUES_COUNT", Cfg().GetAccountSettingsDefaults().GetMaxQueuesCount())
.Utf8("USER_NAME", QueuePath_.UserName);
-
+
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
- }
-}
-
+ }
+}
+
STATEFN(TCreateQueueSchemaActorV2::Preamble) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TSqsEvents::TEvQueueId, HandleQueueId);
hFunc(TSqsEvents::TEvExecuted, OnReadQueueParams);
hFunc(TEvQuota::TEvClearance, OnCreateQueueQuota);
hFunc(TSqsEvents::TEvAtomicCounterIncrementResult, OnAtomicCounterIncrement);
cFunc(TEvPoisonPill::EventType, PassAway);
- }
-}
-
+ }
+}
+
void TCreateQueueSchemaActorV2::HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev) {
THolder<TSqsEvents::TEvQueueCreated> resp;
- if (ev->Get()->Failed) {
+ if (ev->Get()->Failed) {
RLOG_SQS_WARN("Get queue id failed");
resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
- } else if (!ev->Get()->Exists) {
+ } else if (!ev->Get()->Exists) {
resp = MakeErrorResponse(NErrors::VALIDATION_ERROR);
resp->Error = "Target DLQ does not exist";
- } else {
+ } else {
RequestQueueParams();
- return;
- }
-
+ return;
+ }
+
Y_VERIFY(resp);
Send(Sender_, std::move(resp));
PassAway();
-}
-
+}
+
void TCreateQueueSchemaActorV2::OnReadQueueParams(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const auto status = record.GetStatus();
-
+ const auto& record = ev->Get()->Record;
+ const auto status = record.GetStatus();
+
THolder<TSqsEvents::TEvQueueCreated> resp;
-
- if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
- const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
- if (bool(val["exists"])) {
- if (IsCloudMode_) {
- ExistingQueueResourceId_ = TString(val["resourceId"]);
- }
- const ui64 currentVersion = ui64(val["version"]);
+
+ if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
+ const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
+ if (bool(val["exists"])) {
+ if (IsCloudMode_) {
+ ExistingQueueResourceId_ = TString(val["resourceId"]);
+ }
+ const ui64 currentVersion = ui64(val["version"]);
MatchQueueAttributes(currentVersion);
- return;
- } else {
- if (bool(val["overLimit"])) {
+ return;
+ } else {
+ if (bool(val["overLimit"])) {
resp = MakeErrorResponse(NErrors::OVER_LIMIT);
resp->Error = "Too many queues.";
- } else {
+ } else {
if (Cfg().GetQuotingConfig().GetEnableQuoting() && QuoterResources_) {
RequestCreateQueueQuota();
} else {
RunAtomicCounterIncrement();
}
- return;
- }
- }
- } else {
+ return;
+ }
+ }
+ } else {
resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
resp->Error = "Failed to read queue params.";
- }
-
+ }
+
Y_VERIFY(resp);
Send(Sender_, std::move(resp));
PassAway();
-}
-
+}
+
void TCreateQueueSchemaActorV2::RequestCreateQueueQuota() {
TDuration deadline = TDuration::Max();
const auto& quotingConfig = Cfg().GetQuotingConfig();
@@ -393,24 +393,24 @@ void TCreateQueueSchemaActorV2::RunAtomicCounterIncrement() {
}
void TCreateQueueSchemaActorV2::OnAtomicCounterIncrement(TSqsEvents::TEvAtomicCounterIncrementResult::TPtr& ev) {
- auto event = ev->Get();
- if (event->Success) {
- Become(&TCreateQueueSchemaActorV2::CreateComponentsState);
- Version_ = event->NewValue;
- VersionName_ = "v" + ToString(Version_); // add "v" prefix to provide the difference with deprecated version shards
- VersionedQueueFullPath_ = TString::Join(QueuePath_.GetQueuePath(), '/', VersionName_);
+ auto event = ev->Get();
+ if (event->Success) {
+ Become(&TCreateQueueSchemaActorV2::CreateComponentsState);
+ Version_ = event->NewValue;
+ VersionName_ = "v" + ToString(Version_); // add "v" prefix to provide the difference with deprecated version shards
+ VersionedQueueFullPath_ = TString::Join(QueuePath_.GetQueuePath(), '/', VersionName_);
CreateComponents();
- return;
- } else {
+ return;
+ } else {
auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
resp->Error = "Failed to create unique id.";
- resp->State = EQueueState::Creating;
+ resp->State = EQueueState::Creating;
Send(Sender_, std::move(resp));
- }
-
+ }
+
PassAway();
-}
-
+}
+
static const char* const GetTablesFormatQuery = R"__(
(
(let defaultTablesFormat (Parameter 'DEFAULT_TABLES_FORMAT (DataType 'Uint32)))
@@ -446,126 +446,126 @@ void TCreateQueueSchemaActorV2::RequestTablesFormatSettings(const TString& accou
}
void TCreateQueueSchemaActorV2::RegisterMakeDirActor(const TString& workingDir, const TString& dirName) {
- auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
-
- trans->SetWorkingDir(workingDir);
+ auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
+
+ trans->SetWorkingDir(workingDir);
trans->SetOperationType(NKikimrSchemeOp::ESchemeOpMkDir);
- trans->MutableMkDir()->SetName(dirName);
-
+ trans->MutableMkDir()->SetName(dirName);
+
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
-}
-
+}
+
void TCreateQueueSchemaActorV2::RequestLeaderTabletId() {
RLOG_SQS_TRACE("Requesting leader tablet id for path id " << TableWithLeaderPathId_.second);
- THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate());
+ THolder<TEvTxUserProxy::TEvNavigate> request(new TEvTxUserProxy::TEvNavigate());
request->Record.MutableDescribePath()->SetSchemeshardId(TableWithLeaderPathId_.first);
request->Record.MutableDescribePath()->SetPathId(TableWithLeaderPathId_.second);
Send(MakeTxProxyID(), std::move(request));
-}
-
+}
+
void TCreateQueueSchemaActorV2::CreateComponents() {
- switch (CurrentCreationStep_) {
+ switch (CurrentCreationStep_) {
case ECreateComponentsStep::GetTablesFormatSetting: {
RequestTablesFormatSettings(QueuePath_.UserName);
break;
}
- case ECreateComponentsStep::MakeQueueDir: {
+ case ECreateComponentsStep::MakeQueueDir: {
RegisterMakeDirActor(QueuePath_.GetUserPath(), QueuePath_.QueueName);
- break;
- }
- case ECreateComponentsStep::MakeQueueVersionDir: {
+ break;
+ }
+ case ECreateComponentsStep::MakeQueueVersionDir: {
RegisterMakeDirActor(QueuePath_.GetQueuePath(), VersionName_);
- break;
- }
- case ECreateComponentsStep::MakeShards: {
- for (ui64 shardIdx = 0; shardIdx < RequiredShardsCount_; ++shardIdx) {
+ break;
+ }
+ case ECreateComponentsStep::MakeShards: {
+ for (ui64 shardIdx = 0; shardIdx < RequiredShardsCount_; ++shardIdx) {
RegisterMakeDirActor(VersionedQueueFullPath_, ToString(shardIdx));
- }
- break;
- }
- case ECreateComponentsStep::MakeTables: {
- for (const TTable& table : RequiredTables_) {
- auto ev = MakeCreateTableEvent(VersionedQueueFullPath_, table, RequiredShardsCount_);
- auto* cmd = ev->Record.MutableTransaction()->MutableModifyScheme()->MutableCreateTable();
- cmd->MutablePartitionConfig()->MutablePipelineConfig()->SetEnableOutOfOrder(Request_.GetEnableOutOfOrderTransactionsExecution());
-
+ }
+ break;
+ }
+ case ECreateComponentsStep::MakeTables: {
+ for (const TTable& table : RequiredTables_) {
+ auto ev = MakeCreateTableEvent(VersionedQueueFullPath_, table, RequiredShardsCount_);
+ auto* cmd = ev->Record.MutableTransaction()->MutableModifyScheme()->MutableCreateTable();
+ cmd->MutablePartitionConfig()->MutablePipelineConfig()->SetEnableOutOfOrder(Request_.GetEnableOutOfOrderTransactionsExecution());
+
const TActorId actorId = Register(new TMiniKqlExecutionActor(
SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
-
+
if (table.HasLeaderTablet && !CreateTableWithLeaderTabletActorId_) {
CreateTableWithLeaderTabletActorId_ = actorId;
- }
- }
-
- break;
- }
+ }
+ }
+
+ break;
+ }
case ECreateComponentsStep::DescribeTableForSetSchemeShardId: {
SendDescribeTable();
break;
}
case ECreateComponentsStep::DiscoverLeaderTabletId: {
RequestLeaderTabletId();
- break;
- }
+ break;
+ }
case ECreateComponentsStep::AddQuoterResource: {
AddRPSQuota();
break;
}
- }
-}
-
+ }
+}
+
STATEFN(TCreateQueueSchemaActorV2::CreateComponentsState) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TSqsEvents::TEvExecuted, OnExecuted);
hFunc(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult, OnDescribeSchemeResult);
hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleTableDescription);
hFunc(NKesus::TEvKesus::TEvAddQuoterResourceResult, HandleAddQuoterResource);
cFunc(TEvPoisonPill::EventType, PassAway);
- }
-}
-
+ }
+}
+
void TCreateQueueSchemaActorV2::Step() {
RLOG_SQS_TRACE("Next step. Step: " << (int)CurrentCreationStep_);
- switch (CurrentCreationStep_) {
+ switch (CurrentCreationStep_) {
case ECreateComponentsStep::GetTablesFormatSetting: {
CurrentCreationStep_ = ECreateComponentsStep::MakeQueueDir;
break;
}
- case ECreateComponentsStep::MakeQueueDir: {
+ case ECreateComponentsStep::MakeQueueDir: {
if (TablesFormat_ == 0) {
CurrentCreationStep_ = ECreateComponentsStep::MakeQueueVersionDir;
} else {
CurrentCreationStep_ = ECreateComponentsStep::DescribeTableForSetSchemeShardId;
}
- break;
- }
- case ECreateComponentsStep::MakeQueueVersionDir: {
- if (IsFifo_) {
- CurrentCreationStep_ = ECreateComponentsStep::MakeTables;
- } else {
- CurrentCreationStep_ = ECreateComponentsStep::MakeShards;
- }
- break;
- }
- case ECreateComponentsStep::MakeShards: {
- if (++CreatedShardsCount_ != RequiredShardsCount_) {
- return; // do not progress
- }
-
- CurrentCreationStep_ = ECreateComponentsStep::MakeTables;
- break;
- }
-
- case ECreateComponentsStep::MakeTables: {
- if (++CreatedTablesCount_ != RequiredTables_.size()) {
- return; // do not progress
- }
-
+ break;
+ }
+ case ECreateComponentsStep::MakeQueueVersionDir: {
+ if (IsFifo_) {
+ CurrentCreationStep_ = ECreateComponentsStep::MakeTables;
+ } else {
+ CurrentCreationStep_ = ECreateComponentsStep::MakeShards;
+ }
+ break;
+ }
+ case ECreateComponentsStep::MakeShards: {
+ if (++CreatedShardsCount_ != RequiredShardsCount_) {
+ return; // do not progress
+ }
+
+ CurrentCreationStep_ = ECreateComponentsStep::MakeTables;
+ break;
+ }
+
+ case ECreateComponentsStep::MakeTables: {
+ if (++CreatedTablesCount_ != RequiredTables_.size()) {
+ return; // do not progress
+ }
+
Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second);
CurrentCreationStep_ = ECreateComponentsStep::DiscoverLeaderTabletId;
- break;
- }
+ break;
+ }
case ECreateComponentsStep::DescribeTableForSetSchemeShardId: {
Y_VERIFY(TablesFormat_ == 1);
Y_VERIFY(TableWithLeaderPathId_.first && TableWithLeaderPathId_.second);
@@ -578,28 +578,28 @@ void TCreateQueueSchemaActorV2::Step() {
break;
}
case ECreateComponentsStep::AddQuoterResource: {
- Y_VERIFY(false); // unreachable
+ Y_VERIFY(false); // unreachable
break;
- }
- }
-
+ }
+ }
+
CreateComponents();
-}
-
+}
+
void TCreateQueueSchemaActorV2::OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const auto status = record.GetStatus();
+ const auto& record = ev->Get()->Record;
+ const auto status = record.GetStatus();
RLOG_SQS_TRACE("OnExecuted: " << ev->Get()->Record);
-
+
if (ev->Sender == CreateTableWithLeaderTabletActorId_) {
CreateTableWithLeaderTabletTxId_ = record.GetTxId();
TableWithLeaderPathId_ = std::make_pair(record.GetSchemeShardTabletId(), record.GetPathId());
RLOG_SQS_TRACE("Handle executed transaction with leader tablet: " << record);
- }
-
- // Note:
- // SS finishes transaction immediately if the specified path already exists
- // DO NOT add any special logic based on the result type (except for an error)
+ }
+
+ // Note:
+ // SS finishes transaction immediately if the specified path already exists
+ // DO NOT add any special logic based on the result type (except for an error)
if (IsGoodStatusCode(status)) {
if (CurrentCreationStep_ == ECreateComponentsStep::GetTablesFormatSetting) {
const TValue value(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
@@ -624,49 +624,49 @@ void TCreateQueueSchemaActorV2::OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
}
Step();
- } else {
+ } else {
RLOG_SQS_WARN("Component creation request execution error: " << record);
-
+
auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
- resp->State = EQueueState::Creating;
-
- if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::WrongRequest) {
+ resp->State = EQueueState::Creating;
+
+ if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::WrongRequest) {
resp->Error = TStringBuilder() << "Missing account: " << QueuePath_.UserName << ".";
- } else {
- resp->Error = record.GetMiniKQLErrors();
- }
-
+ } else {
+ resp->Error = record.GetMiniKQLErrors();
+ }
+
Send(Sender_, std::move(resp));
-
+
PassAway();
- }
-}
-
+ }
+}
+
void TCreateQueueSchemaActorV2::OnDescribeSchemeResult(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) {
RLOG_SQS_TRACE("OnDescribeSchemeResult for leader tablet: " << ev->Get()->GetRecord());
- const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription();
-
+ const auto& pathDescription = ev->Get()->GetRecord().GetPathDescription();
+
if (ev->Get()->GetRecord().GetStatus() != NKikimrScheme::StatusSuccess || pathDescription.TablePartitionsSize() == 0 || !pathDescription.GetTablePartitions(0).GetDatashardId()) {
- // fail
+ // fail
auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
- resp->State = EQueueState::Creating;
+ resp->State = EQueueState::Creating;
resp->Error = "Failed to discover leader.";
-
+
Send(Sender_, std::move(resp));
-
+
PassAway();
- return;
- }
-
+ return;
+ }
+
LeaderTabletId_ = pathDescription.GetTablePartitions(0).GetDatashardId();
-
+
if (Cfg().GetQuotingConfig().GetEnableQuoting() && Cfg().GetQuotingConfig().HasKesusQuoterConfig()) {
Step();
} else {
CommitNewVersion();
}
-}
-
+}
+
void TCreateQueueSchemaActorV2::SendDescribeTable() {
auto navigateRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>();
@@ -718,38 +718,38 @@ void TCreateQueueSchemaActorV2::HandleAddQuoterResource(NKesus::TEvKesus::TEvAdd
}
}
-static const char* const CommitQueueParamsQuery = R"__(
- (
- (let name (Parameter 'NAME (DataType 'Utf8String)))
- (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String)))
- (let folderId (Parameter 'FOLDERID (DataType 'Utf8String)))
- (let id (Parameter 'ID (DataType 'String)))
- (let fifo (Parameter 'FIFO (DataType 'Bool)))
+static const char* const CommitQueueParamsQuery = R"__(
+ (
+ (let name (Parameter 'NAME (DataType 'Utf8String)))
+ (let customName (Parameter 'CUSTOMNAME (DataType 'Utf8String)))
+ (let folderId (Parameter 'FOLDERID (DataType 'Utf8String)))
+ (let id (Parameter 'ID (DataType 'String)))
+ (let fifo (Parameter 'FIFO (DataType 'Bool)))
(let contentBasedDeduplication (Parameter 'CONTENT_BASED_DEDUPLICATION (DataType 'Bool)))
- (let now (Parameter 'NOW (DataType 'Uint64)))
- (let shards (Parameter 'SHARDS (DataType 'Uint64)))
- (let partitions (Parameter 'PARTITIONS (DataType 'Uint64)))
+ (let now (Parameter 'NOW (DataType 'Uint64)))
+ (let shards (Parameter 'SHARDS (DataType 'Uint64)))
+ (let partitions (Parameter 'PARTITIONS (DataType 'Uint64)))
(let masterTabletId (Parameter 'MASTER_TABLET_ID (DataType 'Uint64)))
(let tablesFormat (Parameter 'TABLES_FORMAT (DataType 'Uint32)))
- (let version (Parameter 'VERSION (DataType 'Uint64)))
+ (let version (Parameter 'VERSION (DataType 'Uint64)))
(let queueIdNumberHash (Parameter 'QUEUE_ID_NUMBER_HASH (DataType 'Uint64)))
- (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64)))
- (let delay (Parameter 'DELAY (DataType 'Uint64)))
- (let visibility (Parameter 'VISIBILITY (DataType 'Uint64)))
- (let retention (Parameter 'RETENTION (DataType 'Uint64)))
+ (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64)))
+ (let delay (Parameter 'DELAY (DataType 'Uint64)))
+ (let visibility (Parameter 'VISIBILITY (DataType 'Uint64)))
+ (let retention (Parameter 'RETENTION (DataType 'Uint64)))
(let receiveMessageWaitTime (Parameter 'RECEIVE_MESSAGE_WAIT_TIME (DataType 'Uint64)))
- (let dlqArn (Parameter 'DLQ_TARGET_ARN (DataType 'Utf8String)))
- (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String)))
- (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64)))
+ (let dlqArn (Parameter 'DLQ_TARGET_ARN (DataType 'Utf8String)))
+ (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String)))
+ (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64)))
(let defaultMaxQueuesCount (Parameter 'DEFAULT_MAX_QUEUES_COUNT (DataType 'Uint64)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
-
+
(let attrsTable '%1$s/Attributes)
(let stateTable '%1$s/State)
(let settingsTable '%2$s/.Settings)
(let queuesTable '%2$s/.Queues)
(let eventsTable '%2$s/.Events)
-
+
(let maxQueuesCountSettingRow '(
'('Account userName)
'('Name (Utf8String '"MaxQueuesCount"))))
@@ -760,15 +760,15 @@ static const char* const CommitQueueParamsQuery = R"__(
(let queuesRange '(
'('Account userName userName)
- '('QueueName (Utf8String '"") (Void))))
- (let queues
+ '('QueueName (Utf8String '"") (Void))))
+ (let queues
(Member (SelectRange queuesTable queuesRange '('QueueName 'CustomQueueName 'Version 'FolderId 'QueueState) '()) 'List))
- (let overLimit
+ (let overLimit
(LessOrEqual maxQueuesCountSetting (Length queues)))
-
+
(let queuesRow '(
'('Account userName)
- '('QueueName name)))
+ '('QueueName name)))
(let eventsRow '(
'('Account userName)
@@ -776,66 +776,66 @@ static const char* const CommitQueueParamsQuery = R"__(
'('EventType (Uint64 '1))))
(let queuesSelect '(
- 'QueueState
- 'QueueId
- 'FifoQueue
- 'Shards
- 'Partitions
- 'Version))
+ 'QueueState
+ 'QueueId
+ 'FifoQueue
+ 'Shards
+ 'Partitions
+ 'Version))
(let queuesRead (SelectRow queuesTable queuesRow queuesSelect))
-
- (let existingQueuesWithSameNameAndFolderId
+
+ (let existingQueuesWithSameNameAndFolderId
(If (Equal (Utf8String '"") customName)
(List (TypeOf queues))
- (Filter queues (lambda '(item) (block '(
- (return (Coalesce
- (And
- (Equal (Member item 'CustomQueueName) customName)
- (Equal (Member item 'FolderId) folderId))
- (Bool 'false))))))
- )
- )
- )
-
- (let existingResourceId (Coalesce
- (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName)
- (Utf8String '"")
- ))
-
- (let queueExists
- (Coalesce
- (Or
- (Or
+ (Filter queues (lambda '(item) (block '(
+ (return (Coalesce
+ (And
+ (Equal (Member item 'CustomQueueName) customName)
+ (Equal (Member item 'FolderId) folderId))
+ (Bool 'false))))))
+ )
+ )
+ )
+
+ (let existingResourceId (Coalesce
+ (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'QueueName)
+ (Utf8String '"")
+ ))
+
+ (let queueExists
+ (Coalesce
+ (Or
+ (Or
(Equal (Uint64 '1) (Member queuesRead 'QueueState))
(Equal (Uint64 '3) (Member queuesRead 'QueueState))
- )
- (NotEqual (Utf8String '"") existingResourceId)
- )
- (Bool 'false)))
-
- (let currentVersion
- (Coalesce
- (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version)
+ )
+ (NotEqual (Utf8String '"") existingResourceId)
+ )
+ (Bool 'false)))
+
+ (let currentVersion
+ (Coalesce
+ (Member (ToOptional existingQueuesWithSameNameAndFolderId) 'Version)
(Member queuesRead 'Version)
- (Uint64 '0)
- )
- )
-
+ (Uint64 '0)
+ )
+ )
+
(let queuesUpdate '(
- '('QueueId id)
- '('CustomQueueName customName)
- '('FolderId folderId)
- '('QueueState (Uint64 '3))
- '('FifoQueue fifo)
- '('DeadLetterQueue (Bool 'false))
- '('CreatedTimestamp now)
- '('Shards shards)
- '('Partitions partitions)
- '('Version version)
- '('DlqName dlqName)
+ '('QueueId id)
+ '('CustomQueueName customName)
+ '('FolderId folderId)
+ '('QueueState (Uint64 '3))
+ '('FifoQueue fifo)
+ '('DeadLetterQueue (Bool 'false))
+ '('CreatedTimestamp now)
+ '('Shards shards)
+ '('Partitions partitions)
+ '('Version version)
+ '('DlqName dlqName)
'('MasterTabletId masterTabletId)
'('TablesFormat tablesFormat)))
-
+
(let eventsUpdate '(
'('CustomQueueName customName)
'('EventTimestamp now)
@@ -843,23 +843,23 @@ static const char* const CommitQueueParamsQuery = R"__(
(let attrRow '(%3$s))
- (let attrUpdate '(
+ (let attrUpdate '(
'('ContentBasedDeduplication contentBasedDeduplication)
- '('DelaySeconds delay)
- '('FifoQueue fifo)
- '('MaximumMessageSize maxSize)
- '('MessageRetentionPeriod retention)
+ '('DelaySeconds delay)
+ '('FifoQueue fifo)
+ '('MaximumMessageSize maxSize)
+ '('MessageRetentionPeriod retention)
'('ReceiveMessageWaitTime receiveMessageWaitTime)
- '('MaxReceiveCount maxReceiveCount)
- '('DlqArn dlqArn)
- '('DlqName dlqName)
- '('VisibilityTimeout visibility)))
-
- (let willCommit
- (And
- (Not queueExists)
- (Not overLimit)))
-
+ '('MaxReceiveCount maxReceiveCount)
+ '('DlqArn dlqArn)
+ '('DlqName dlqName)
+ '('VisibilityTimeout visibility)))
+
+ (let willCommit
+ (And
+ (Not queueExists)
+ (Not overLimit)))
+
(let stateUpdate '(
'('CleanupTimestamp now)
'('CreatedTimestamp now)
@@ -873,19 +873,19 @@ static const char* const CommitQueueParamsQuery = R"__(
(let queueIdNumberAndShardHashes (AsList %4$s))
- (return (Extend
- (AsList
- (SetResult 'exists queueExists)
- (SetResult 'overLimit overLimit)
- (SetResult 'version currentVersion)
- (SetResult 'resourceId existingResourceId)
- (SetResult 'commited willCommit))
-
+ (return (Extend
+ (AsList
+ (SetResult 'exists queueExists)
+ (SetResult 'overLimit overLimit)
+ (SetResult 'version currentVersion)
+ (SetResult 'resourceId existingResourceId)
+ (SetResult 'commited willCommit))
+
(ListIf queueExists (SetResult 'meta queuesRead))
-
+
(ListIf willCommit (UpdateRow queuesTable queuesRow queuesUpdate))
(ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate))
- (ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate))
+ (ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate))
(If (Not willCommit) (AsList (Void))
(Map (Enumerate queueIdNumberAndShardHashes) (lambda '(item) (block '(
@@ -894,21 +894,21 @@ static const char* const CommitQueueParamsQuery = R"__(
(let queueIdNumberAndShardHash (Nth item '1))
(let row '(%5$s))
- (let update '(
- '('CleanupTimestamp now)
- '('CreatedTimestamp now)
- '('LastModifiedTimestamp now)
- '('InflyCount (Int64 '0))
- '('MessageCount (Int64 '0))
- '('RetentionBoundary (Uint64 '0))
- '('ReadOffset (Uint64 '0))
- '('WriteOffset (Uint64 '0))
+ (let update '(
+ '('CleanupTimestamp now)
+ '('CreatedTimestamp now)
+ '('LastModifiedTimestamp now)
+ '('InflyCount (Int64 '0))
+ '('MessageCount (Int64 '0))
+ '('RetentionBoundary (Uint64 '0))
+ '('ReadOffset (Uint64 '0))
+ '('WriteOffset (Uint64 '0))
'('CleanupVersion (Uint64 '0))))
- (return (UpdateRow stateTable row update)))))))
- ))
- )
-)__";
-
+ (return (UpdateRow stateTable row update)))))))
+ ))
+ )
+)__";
+
TString GetStateTableKeys(ui32 tablesFormat, bool isFifo) {
if (tablesFormat == 1) {
if (isFifo) {
@@ -946,8 +946,8 @@ TString GetQueueIdAndShardHashesList(ui64 version, ui32 shards) {
}
void TCreateQueueSchemaActorV2::CommitNewVersion() {
- Become(&TCreateQueueSchemaActorV2::FinalizeAndCommit);
-
+ Become(&TCreateQueueSchemaActorV2::FinalizeAndCommit);
+
TString queuePath;
if (TablesFormat_ == 0) {
queuePath = Join("/", QueuePath_.GetUserPath(), QueuePath_.QueueName, VersionName_);
@@ -965,271 +965,271 @@ void TCreateQueueSchemaActorV2::CommitNewVersion() {
);
auto ev = MakeExecuteEvent(query);
- auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
+ auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
Y_VERIFY(LeaderTabletId_ != 0);
- TParameters(trans->MutableParams()->MutableProto())
- .Utf8("NAME", QueuePath_.QueueName)
- .Utf8("CUSTOMNAME", CustomQueueName_)
- .Utf8("FOLDERID", FolderId_)
- .String("ID", GeneratedQueueId_)
- .Bool("FIFO", IsFifo_)
- .Bool("CONTENT_BASED_DEDUPLICATION", *ValidatedAttributes_.ContentBasedDeduplication)
- .Uint64("NOW", QueueCreationTimestamp_.MilliSeconds())
- .Uint64("SHARDS", RequiredShardsCount_)
- .Uint64("PARTITIONS", Request_.GetPartitions())
+ TParameters(trans->MutableParams()->MutableProto())
+ .Utf8("NAME", QueuePath_.QueueName)
+ .Utf8("CUSTOMNAME", CustomQueueName_)
+ .Utf8("FOLDERID", FolderId_)
+ .String("ID", GeneratedQueueId_)
+ .Bool("FIFO", IsFifo_)
+ .Bool("CONTENT_BASED_DEDUPLICATION", *ValidatedAttributes_.ContentBasedDeduplication)
+ .Uint64("NOW", QueueCreationTimestamp_.MilliSeconds())
+ .Uint64("SHARDS", RequiredShardsCount_)
+ .Uint64("PARTITIONS", Request_.GetPartitions())
.Uint64("MASTER_TABLET_ID", LeaderTabletId_)
.Uint32("TABLES_FORMAT", TablesFormat_)
- .Uint64("VERSION", Version_)
+ .Uint64("VERSION", Version_)
.Uint64("QUEUE_ID_NUMBER_HASH", GetHash(Version_))
- .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize)
- .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds))
- .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout))
- .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod))
- .Uint64("RECEIVE_MESSAGE_WAIT_TIME", SecondsToMs(*ValidatedAttributes_.ReceiveMessageWaitTimeSeconds))
- .Utf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn ? *ValidatedAttributes_.RedrivePolicy.TargetArn : "")
- .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "")
+ .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize)
+ .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds))
+ .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout))
+ .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod))
+ .Uint64("RECEIVE_MESSAGE_WAIT_TIME", SecondsToMs(*ValidatedAttributes_.ReceiveMessageWaitTimeSeconds))
+ .Utf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn ? *ValidatedAttributes_.RedrivePolicy.TargetArn : "")
+ .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "")
.Uint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount ? *ValidatedAttributes_.RedrivePolicy.MaxReceiveCount : 0)
.Uint64("DEFAULT_MAX_QUEUES_COUNT", Cfg().GetAccountSettingsDefaults().GetMaxQueuesCount())
.Utf8("USER_NAME", QueuePath_.UserName);
-
+
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
-}
-
+}
+
STATEFN(TCreateQueueSchemaActorV2::FinalizeAndCommit) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TSqsEvents::TEvExecuted, OnCommit);
cFunc(TEvPoisonPill::EventType, PassAway);
- }
-}
-
+ }
+}
+
void TCreateQueueSchemaActorV2::OnCommit(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const auto status = record.GetStatus();
-
+ const auto& record = ev->Get()->Record;
+ const auto status = record.GetStatus();
+
auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
-
- if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
- const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
- if (bool(val["commited"])) {
- // a new born queue is here!
- resp->QueueId = GeneratedQueueId_;
- resp->Success = true;
+
+ if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
+ const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
+ if (bool(val["commited"])) {
+ // a new born queue is here!
+ resp->QueueId = GeneratedQueueId_;
+ resp->Success = true;
resp->ErrorClass = nullptr;
- } else {
- // something is off
- if (bool(val["overLimit"])) {
+ } else {
+ // something is off
+ if (bool(val["overLimit"])) {
resp->ErrorClass = &NErrors::OVER_LIMIT;
resp->Error = "Too many queues.";
- } else if (bool(val["exists"])) {
- if (IsCloudMode_) {
- ExistingQueueResourceId_ = TString(val["resourceId"]);
- }
- const ui64 currentVersion = ui64(val["version"]);
+ } else if (bool(val["exists"])) {
+ if (IsCloudMode_) {
+ ExistingQueueResourceId_ = TString(val["resourceId"]);
+ }
+ const ui64 currentVersion = ui64(val["version"]);
MatchQueueAttributes(currentVersion);
- return;
- } else {
- Y_VERIFY(false); // unreachable
- }
- }
- } else {
+ return;
+ } else {
+ Y_VERIFY(false); // unreachable
+ }
+ }
+ } else {
resp->Error = "Failed to commit new queue version.";
- }
-
+ }
+
Send(Sender_, std::move(resp));
PassAway();
-}
-
-static const char* const MatchQueueAttributesQuery = R"__(
- (
- (let name (Parameter 'NAME (DataType 'Utf8String)))
- (let fifo (Parameter 'FIFO (DataType 'Bool)))
- (let shards (Parameter 'SHARDS (DataType 'Uint64)))
- (let partitions (Parameter 'PARTITIONS (DataType 'Uint64)))
- (let expectedVersion (Parameter 'EXPECTED_VERSION (DataType 'Uint64)))
- (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64)))
- (let delay (Parameter 'DELAY (DataType 'Uint64)))
- (let visibility (Parameter 'VISIBILITY (DataType 'Uint64)))
- (let retention (Parameter 'RETENTION (DataType 'Uint64)))
- (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String)))
- (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64)))
+}
+
+static const char* const MatchQueueAttributesQuery = R"__(
+ (
+ (let name (Parameter 'NAME (DataType 'Utf8String)))
+ (let fifo (Parameter 'FIFO (DataType 'Bool)))
+ (let shards (Parameter 'SHARDS (DataType 'Uint64)))
+ (let partitions (Parameter 'PARTITIONS (DataType 'Uint64)))
+ (let expectedVersion (Parameter 'EXPECTED_VERSION (DataType 'Uint64)))
+ (let maxSize (Parameter 'MAX_SIZE (DataType 'Uint64)))
+ (let delay (Parameter 'DELAY (DataType 'Uint64)))
+ (let visibility (Parameter 'VISIBILITY (DataType 'Uint64)))
+ (let retention (Parameter 'RETENTION (DataType 'Uint64)))
+ (let dlqName (Parameter 'DLQ_TARGET_NAME (DataType 'Utf8String)))
+ (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint64)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
-
- (let attrsTable '%1$s/%2$s/Attributes)
+
+ (let attrsTable '%1$s/%2$s/Attributes)
(let queuesTable '%3$s/.Queues)
-
+
(let queuesRange '(
'('Account userName userName)
- '('QueueName (Utf8String '"") (Void))))
- (let queues
+ '('QueueName (Utf8String '"") (Void))))
+ (let queues
(Member (SelectRange queuesTable queuesRange '('QueueState) '()) 'List))
-
+
(let queuesRow '(
'('Account userName)
- '('QueueName name)))
+ '('QueueName name)))
(let queuesSelect '(
- 'QueueState
- 'QueueId
- 'FifoQueue
- 'Shards
- 'Partitions
- 'DlqName
- 'Version))
+ 'QueueState
+ 'QueueId
+ 'FifoQueue
+ 'Shards
+ 'Partitions
+ 'DlqName
+ 'Version))
(let queuesRead (SelectRow queuesTable queuesRow queuesSelect))
-
- (let queueExists
- (Coalesce
- (Or
+
+ (let queueExists
+ (Coalesce
+ (Or
(Equal (Uint64 '1) (Member queuesRead 'QueueState))
(Equal (Uint64 '3) (Member queuesRead 'QueueState))
- )
- (Bool 'false)))
-
- (let currentVersion
- (Coalesce
+ )
+ (Bool 'false)))
+
+ (let currentVersion
+ (Coalesce
(Member queuesRead 'Version)
- (Uint64 '0)
- )
- )
-
- (let sameParams
- (Coalesce
- (And
- (And
+ (Uint64 '0)
+ )
+ )
+
+ (let sameParams
+ (Coalesce
+ (And
+ (And
(And (Equal (Member queuesRead 'Shards) shards)
(Equal (Member queuesRead 'Partitions) partitions))
(Equal (Member queuesRead 'FifoQueue) fifo))
(Equal (Coalesce (Member queuesRead 'DlqName) (Utf8String '"")) dlqName))
- (Bool 'true)))
-
- (let attrRow '(
- '('State (Uint64 '0))))
- (let attrSelect '(
- 'DelaySeconds
- 'MaximumMessageSize
- 'MessageRetentionPeriod
- 'MaxReceiveCount
- 'VisibilityTimeout))
- (let attrRead (SelectRow attrsTable attrRow attrSelect))
-
- (let sameAttributes
- (Coalesce
- (And
- (And
- (And (Equal (Member attrRead 'DelaySeconds) delay)
- (And (Equal (Member attrRead 'MaximumMessageSize) maxSize)
- (Equal (Member attrRead 'MessageRetentionPeriod) retention)))
- (Equal (Member attrRead 'VisibilityTimeout) visibility))
- (Equal (Coalesce (Member attrRead 'MaxReceiveCount) (Uint64 '0)) maxReceiveCount))
- (Bool 'true)))
-
- (let sameVersion
- (Equal currentVersion expectedVersion))
-
- (let isSame
- (And
- queueExists
- (And
- sameVersion
- (And
- sameAttributes
- sameParams))))
-
- (let existingQueueId
- (Coalesce
+ (Bool 'true)))
+
+ (let attrRow '(
+ '('State (Uint64 '0))))
+ (let attrSelect '(
+ 'DelaySeconds
+ 'MaximumMessageSize
+ 'MessageRetentionPeriod
+ 'MaxReceiveCount
+ 'VisibilityTimeout))
+ (let attrRead (SelectRow attrsTable attrRow attrSelect))
+
+ (let sameAttributes
+ (Coalesce
+ (And
+ (And
+ (And (Equal (Member attrRead 'DelaySeconds) delay)
+ (And (Equal (Member attrRead 'MaximumMessageSize) maxSize)
+ (Equal (Member attrRead 'MessageRetentionPeriod) retention)))
+ (Equal (Member attrRead 'VisibilityTimeout) visibility))
+ (Equal (Coalesce (Member attrRead 'MaxReceiveCount) (Uint64 '0)) maxReceiveCount))
+ (Bool 'true)))
+
+ (let sameVersion
+ (Equal currentVersion expectedVersion))
+
+ (let isSame
+ (And
+ queueExists
+ (And
+ sameVersion
+ (And
+ sameAttributes
+ sameParams))))
+
+ (let existingQueueId
+ (Coalesce
(Member queuesRead 'QueueId)
- (String '"")))
-
- (return (AsList
- (SetResult 'exists queueExists)
- (SetResult 'sameVersion sameVersion)
- (SetResult 'id existingQueueId)
- (SetResult 'isSame isSame)))
- )
-)__";
-
+ (String '"")))
+
+ (return (AsList
+ (SetResult 'exists queueExists)
+ (SetResult 'sameVersion sameVersion)
+ (SetResult 'id existingQueueId)
+ (SetResult 'isSame isSame)))
+ )
+)__";
+
void TCreateQueueSchemaActorV2::MatchQueueAttributes(const ui64 currentVersion) {
- Become(&TCreateQueueSchemaActorV2::MatchAttributes);
-
- TString versionedQueuePath = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName;
- if (currentVersion != 0) {
- // modern-way constructed queue requires version suffix
- versionedQueuePath = TString::Join(versionedQueuePath, "/v", ToString(currentVersion));
- }
- auto ev = MakeExecuteEvent(Sprintf(
+ Become(&TCreateQueueSchemaActorV2::MatchAttributes);
+
+ TString versionedQueuePath = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName;
+ if (currentVersion != 0) {
+ // modern-way constructed queue requires version suffix
+ versionedQueuePath = TString::Join(versionedQueuePath, "/v", ToString(currentVersion));
+ }
+ auto ev = MakeExecuteEvent(Sprintf(
MatchQueueAttributesQuery, QueuePath_.GetUserPath().c_str(), versionedQueuePath.c_str(), Cfg().GetRoot().c_str()
- ));
- auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
- TParameters(trans->MutableParams()->MutableProto())
- .Utf8("NAME", IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName)
- .Bool("FIFO", IsFifo_)
- .Uint64("SHARDS", RequiredShardsCount_)
- .Uint64("PARTITIONS", Request_.GetPartitions())
- .Uint64("EXPECTED_VERSION", currentVersion)
- .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize)
- .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds))
- .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout))
- .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod))
- .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "")
+ ));
+ auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
+ TParameters(trans->MutableParams()->MutableProto())
+ .Utf8("NAME", IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName)
+ .Bool("FIFO", IsFifo_)
+ .Uint64("SHARDS", RequiredShardsCount_)
+ .Uint64("PARTITIONS", Request_.GetPartitions())
+ .Uint64("EXPECTED_VERSION", currentVersion)
+ .Uint64("MAX_SIZE", *ValidatedAttributes_.MaximumMessageSize)
+ .Uint64("DELAY", SecondsToMs(*ValidatedAttributes_.DelaySeconds))
+ .Uint64("VISIBILITY", SecondsToMs(*ValidatedAttributes_.VisibilityTimeout))
+ .Uint64("RETENTION", SecondsToMs(*ValidatedAttributes_.MessageRetentionPeriod))
+ .Utf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName ? *ValidatedAttributes_.RedrivePolicy.TargetQueueName : "")
.Uint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount ? *ValidatedAttributes_.RedrivePolicy.MaxReceiveCount : 0)
.Utf8("USER_NAME", QueuePath_.UserName);
-
+
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
-}
-
+}
+
STATEFN(TCreateQueueSchemaActorV2::MatchAttributes) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TSqsEvents::TEvExecuted, OnAttributesMatch);
cFunc(TEvPoisonPill::EventType, PassAway);
- }
-}
-
+ }
+}
+
void TCreateQueueSchemaActorV2::OnAttributesMatch(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const auto status = record.GetStatus();
-
+ const auto& record = ev->Get()->Record;
+ const auto status = record.GetStatus();
+
auto resp = MakeErrorResponse(NErrors::INTERNAL_FAILURE);
-
- if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
- const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
- if (bool(val["exists"])) {
- resp->AlreadyExists = true;
+
+ if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
+ const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
+ if (bool(val["exists"])) {
+ resp->AlreadyExists = true;
resp->ErrorClass = nullptr;
- resp->ExistingQueueResourceId = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName;
- const bool isSame = bool(val["isSame"]);
- if (isSame || !EnableQueueAttributesValidation_) {
- resp->Success = true;
+ resp->ExistingQueueResourceId = IsCloudMode_ ? ExistingQueueResourceId_ : QueuePath_.QueueName;
+ const bool isSame = bool(val["isSame"]);
+ if (isSame || !EnableQueueAttributesValidation_) {
+ resp->Success = true;
resp->ErrorClass = nullptr;
- resp->QueueId = TString(val["id"]);
-
- if (!isSame) {
+ resp->QueueId = TString(val["id"]);
+
+ if (!isSame) {
RLOG_SQS_WARN("Queue attributes do not match for account " << QueuePath_.UserName << " and queue name " << QueuePath_.QueueName);
- }
- } else {
+ }
+ } else {
resp->Error = "queue with specified name already exists and has different attributes.";
resp->ErrorClass = &NErrors::VALIDATION_ERROR;
- }
-
+ }
+
if (CurrentCreationStep_ == ECreateComponentsStep::DiscoverLeaderTabletId) {
- // call the special version of cleanup actor
+ // call the special version of cleanup actor
RLOG_SQS_WARN("Removing redundant queue version: " << Version_ << " for queue " <<
- QueuePath_.GetQueuePath() << ". Shards: " << RequiredShardsCount_ << " IsFifo: " << IsFifo_);
+ QueuePath_.GetQueuePath() << ". Shards: " << RequiredShardsCount_ << " IsFifo: " << IsFifo_);
Register(new TDeleteQueueSchemaActorV2(QueuePath_, SelfId(), RequestId_, UserCounters_,
- Version_, RequiredShardsCount_, IsFifo_));
- }
-
- } else {
+ Version_, RequiredShardsCount_, IsFifo_));
+ }
+
+ } else {
resp->Error = "Queue was removed recently.";
resp->ErrorClass = &NErrors::QUEUE_DELETED_RECENTLY;
resp->State = EQueueState::Deleting;
- }
- } else {
+ }
+ } else {
resp->Error = "Failed to compare queue attributes.";
- }
-
+ }
+
Send(Sender_, std::move(resp));
PassAway();
-}
-
+}
+
void TCreateQueueSchemaActorV2::PassAway() {
if (AddQuoterResourceActor_) {
Send(AddQuoterResourceActor_, new TEvPoisonPill());
@@ -1238,97 +1238,97 @@ void TCreateQueueSchemaActorV2::PassAway() {
TActorBootstrapped<TCreateQueueSchemaActorV2>::PassAway();
}
-TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path,
+TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path,
const TActorId& sender,
- const TString& requestId,
+ const TString& requestId,
TIntrusivePtr<TUserCounters> userCounters)
- : QueuePath_(path)
- , Sender_(sender)
- , SI_(0)
- , RequestId_(requestId)
+ : QueuePath_(path)
+ , Sender_(sender)
+ , SI_(0)
+ , RequestId_(requestId)
, UserCounters_(std::move(userCounters))
-{
-}
-
-TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path,
+{
+}
+
+TDeleteQueueSchemaActorV2::TDeleteQueueSchemaActorV2(const TQueuePath& path,
const TActorId& sender,
- const TString& requestId,
+ const TString& requestId,
TIntrusivePtr<TUserCounters> userCounters,
- const ui64 advisedQueueVersion,
- const ui64 advisedShardCount,
- const bool advisedIsFifoFlag)
- : QueuePath_(path)
- , Sender_(sender)
- , SI_(static_cast<ui32>(EDeleting::RemoveTables))
- , RequestId_(requestId)
+ const ui64 advisedQueueVersion,
+ const ui64 advisedShardCount,
+ const bool advisedIsFifoFlag)
+ : QueuePath_(path)
+ , Sender_(sender)
+ , SI_(static_cast<ui32>(EDeleting::RemoveTables))
+ , RequestId_(requestId)
, UserCounters_(std::move(userCounters))
-{
- Y_VERIFY(advisedQueueVersion > 0);
-
- Version_ = advisedQueueVersion;
-
- PrepareCleanupPlan(advisedIsFifoFlag, advisedShardCount);
-}
-
+{
+ Y_VERIFY(advisedQueueVersion > 0);
+
+ Version_ = advisedQueueVersion;
+
+ PrepareCleanupPlan(advisedIsFifoFlag, advisedShardCount);
+}
+
void TDeleteQueueSchemaActorV2::Bootstrap() {
NextAction();
- Become(&TThis::StateFunc);
-}
-
-void TDeleteQueueSchemaActorV2::PrepareCleanupPlan(const bool isFifo, const ui64 shardCount) {
- if (isFifo) {
- Tables_ = GetFifoTables();
- } else {
- Tables_ = GetStandardTableNames(shardCount);
-
- for (ui64 i = 0; i < shardCount; ++i) {
- Shards_.push_back(i);
- }
- }
-}
-
-static TString GetVersionedQueueDir(const TString& baseQueueDir, const ui64 version) {
- if (!version) {
- return baseQueueDir;
- }
-
- return TString::Join(baseQueueDir, "/v", ToString(version));
-}
-
-static const char* EraseQueueRecordQuery = R"__(
- (
- (let name (Parameter 'NAME (DataType 'Utf8String)))
+ Become(&TThis::StateFunc);
+}
+
+void TDeleteQueueSchemaActorV2::PrepareCleanupPlan(const bool isFifo, const ui64 shardCount) {
+ if (isFifo) {
+ Tables_ = GetFifoTables();
+ } else {
+ Tables_ = GetStandardTableNames(shardCount);
+
+ for (ui64 i = 0; i < shardCount; ++i) {
+ Shards_.push_back(i);
+ }
+ }
+}
+
+static TString GetVersionedQueueDir(const TString& baseQueueDir, const ui64 version) {
+ if (!version) {
+ return baseQueueDir;
+ }
+
+ return TString::Join(baseQueueDir, "/v", ToString(version));
+}
+
+static const char* EraseQueueRecordQuery = R"__(
+ (
+ (let name (Parameter 'NAME (DataType 'Utf8String)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
(let now (Parameter 'NOW (DataType 'Uint64)))
-
+
(let queuesTable '%2$s/.Queues)
(let eventsTable '%2$s/.Events)
-
+
(let queuesRow '(
'('Account userName)
- '('QueueName name)))
+ '('QueueName name)))
(let eventsRow '(
'('Account userName)
'('QueueName name)
'('EventType (Uint64 '0))))
(let queuesSelect '(
- 'QueueState
- 'Version
- 'FifoQueue
+ 'QueueState
+ 'Version
+ 'FifoQueue
'Shards
'CustomQueueName
'CreatedTimestamp
'FolderId))
(let queuesRead (SelectRow queuesTable queuesRow queuesSelect))
-
- (let currentVersion
- (Coalesce
+
+ (let currentVersion
+ (Coalesce
(Member queuesRead 'Version)
- (Uint64 '0)
- )
- )
-
+ (Uint64 '0)
+ )
+ )
+
(let queueCreateTs
(Coalesce
(Member queuesRead 'CreatedTimestamp)
@@ -1351,129 +1351,129 @@ static const char* EraseQueueRecordQuery = R"__(
(let eventTs (Max now (Add queueCreateTs (Uint64 '2))))
- (let queueExists
- (Coalesce
- (Or
+ (let queueExists
+ (Coalesce
+ (Or
(Equal (Uint64 '1) (Member queuesRead 'QueueState))
(Equal (Uint64 '3) (Member queuesRead 'QueueState))
- )
- (Bool 'false)))
-
+ )
+ (Bool 'false)))
+
(let eventsUpdate '(
'('CustomQueueName customName)
'('EventTimestamp eventTs)
'('FolderId folderId)))
- (return (AsList
- (SetResult 'exists queueExists)
- (SetResult 'version currentVersion)
+ (return (AsList
+ (SetResult 'exists queueExists)
+ (SetResult 'version currentVersion)
(SetResult 'fields queuesRead)
(If queueExists (UpdateRow eventsTable eventsRow eventsUpdate) (Void))
(If queueExists (EraseRow queuesTable queuesRow) (Void))))
- )
-)__";
-
+ )
+)__";
+
void TDeleteQueueSchemaActorV2::NextAction() {
- switch (EDeleting(SI_)) {
- case EDeleting::EraseQueueRecord: {
+ switch (EDeleting(SI_)) {
+ case EDeleting::EraseQueueRecord: {
auto ev = MakeExecuteEvent(Sprintf(EraseQueueRecordQuery, QueuePath_.GetUserPath().c_str(), Cfg().GetRoot().c_str()));
- auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
+ auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
auto nowMs = TInstant::Now().MilliSeconds();
- TParameters(trans->MutableParams()->MutableProto())
+ TParameters(trans->MutableParams()->MutableProto())
.Utf8("NAME", QueuePath_.QueueName)
.Utf8("USER_NAME", QueuePath_.UserName)
.Uint64("NOW", nowMs);
-
+
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
- break;
- }
- case EDeleting::RemoveTables: {
+ break;
+ }
+ case EDeleting::RemoveTables: {
Y_VERIFY(!Tables_.empty());
-
+
Register(new TMiniKqlExecutionActor(
SelfId(), RequestId_, MakeDeleteTableEvent(GetVersionedQueueDir(QueuePath_, Version_), Tables_.back()), false, QueuePath_, GetTransactionCounters(UserCounters_))
- );
- break;
- }
- case EDeleting::RemoveShards: {
+ );
+ break;
+ }
+ case EDeleting::RemoveShards: {
Register(new TMiniKqlExecutionActor(
SelfId(), RequestId_, MakeRemoveDirectoryEvent(GetVersionedQueueDir(QueuePath_, Version_), ToString(Shards_.back())), false, QueuePath_, GetTransactionCounters(UserCounters_))
- );
- break;
- }
- case EDeleting::RemoveQueueVersionDirectory: {
+ );
+ break;
+ }
+ case EDeleting::RemoveQueueVersionDirectory: {
Register(new TMiniKqlExecutionActor(
SelfId(), RequestId_, MakeRemoveDirectoryEvent(QueuePath_, TString::Join("v", ToString(Version_))), false, QueuePath_, GetTransactionCounters(UserCounters_))
- );
- break;
- }
- case EDeleting::RemoveQueueDirectory: {
- // this may silently fail for versioned queues
+ );
+ break;
+ }
+ case EDeleting::RemoveQueueDirectory: {
+ // this may silently fail for versioned queues
Register(new TMiniKqlExecutionActor(
SelfId(), RequestId_, MakeRemoveDirectoryEvent(QueuePath_.GetUserPath(), QueuePath_.QueueName), false, QueuePath_, GetTransactionCounters(UserCounters_))
- );
- break;
- }
+ );
+ break;
+ }
case EDeleting::DeleteQuoterResource: {
DeleteRPSQuota();
break;
}
- case EDeleting::Finish: {
+ case EDeleting::Finish: {
Send(Sender_, MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, true));
PassAway();
- break;
- }
- }
-}
-
+ break;
+ }
+ }
+}
+
void TDeleteQueueSchemaActorV2::DoSuccessOperation() {
- if (EDeleting(SI_) == EDeleting::RemoveTables) {
- Tables_.pop_back();
-
- if (Tables_.empty()) {
- if (Shards_.empty()) {
- SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory);
- } else {
- SI_ = ui32(EDeleting::RemoveShards);
- }
- }
- } else if (EDeleting(SI_) == EDeleting::RemoveShards) {
- Shards_.pop_back();
-
- if (Shards_.empty()) {
- SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory);
- }
- } else {
- SI_++;
+ if (EDeleting(SI_) == EDeleting::RemoveTables) {
+ Tables_.pop_back();
+
+ if (Tables_.empty()) {
+ if (Shards_.empty()) {
+ SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory);
+ } else {
+ SI_ = ui32(EDeleting::RemoveShards);
+ }
+ }
+ } else if (EDeleting(SI_) == EDeleting::RemoveShards) {
+ Shards_.pop_back();
+
+ if (Shards_.empty()) {
+ SI_ = ui32(Version_ ? EDeleting::RemoveQueueVersionDirectory : EDeleting::RemoveQueueDirectory);
+ }
+ } else {
+ SI_++;
if ((!Cfg().GetQuotingConfig().GetEnableQuoting() || !Cfg().GetQuotingConfig().HasKesusQuoterConfig()) && EDeleting(SI_) == EDeleting::DeleteQuoterResource) {
SI_++;
}
- }
-
+ }
+
NextAction();
-}
-
+}
+
void TDeleteQueueSchemaActorV2::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
+ const auto& record = ev->Get()->Record;
if (IsGoodStatusCode(record.GetStatus())) {
- if (EDeleting(SI_) == EDeleting::EraseQueueRecord) {
- const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
- if (!bool(val["exists"])) {
+ if (EDeleting(SI_) == EDeleting::EraseQueueRecord) {
+ const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
+ if (!bool(val["exists"])) {
Send(Sender_,
- MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, false, "Queue does not exist."));
+ MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, false, "Queue does not exist."));
PassAway();
- return;
- } else {
- Version_ = ui64(val["version"]);
-
- PrepareCleanupPlan(bool(val["fields"]["FifoQueue"]), ui64(val["fields"]["Shards"]));
- }
- }
-
+ return;
+ } else {
+ Version_ = ui64(val["version"]);
+
+ PrepareCleanupPlan(bool(val["fields"]["FifoQueue"]), ui64(val["fields"]["Shards"]));
+ }
+ }
+
DoSuccessOperation();
- } else {
+ } else {
RLOG_SQS_WARN("request execution error: " << record);
-
+
if (EDeleting(SI_) == EDeleting::EraseQueueRecord) {
Send(Sender_,
MakeHolder<TSqsEvents::TEvQueueDeleted>(QueuePath_, false, "Failed to erase queue record."));
@@ -1481,11 +1481,11 @@ void TDeleteQueueSchemaActorV2::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev
return;
}
- // we don't really care if some components are already deleted
+ // we don't really care if some components are already deleted
DoSuccessOperation();
- }
-}
-
+ }
+}
+
void TDeleteQueueSchemaActorV2::DeleteRPSQuota() {
NKikimrKesus::TEvDeleteQuoterResource cmd;
cmd.SetResourcePath(TStringBuilder() << RPS_QUOTA_NAME << "/" << QueuePath_.QueueName);
diff --git a/ydb/core/ymq/actor/queue_schema.h b/ydb/core/ymq/actor/queue_schema.h
index 54cc41530d..793ba1c06a 100644
--- a/ydb/core/ymq/actor/queue_schema.h
+++ b/ydb/core/ymq/actor/queue_schema.h
@@ -1,85 +1,85 @@
-#pragma once
+#pragma once
#include "defs.h"
-
-#include "schema.h"
+
+#include "schema.h"
#include <ydb/core/base/quoter.h>
#include <ydb/core/kesus/tablet/events.h>
#include <ydb/core/protos/config.pb.h>
#include <ydb/public/lib/value/value.h>
#include <ydb/core/ymq/base/queue_attributes.h>
-
-#include <util/generic/maybe.h>
-
+
+#include <util/generic/maybe.h>
+
namespace NKikimr::NSQS {
-
-class TCreateQueueSchemaActorV2
+
+class TCreateQueueSchemaActorV2
: public TActorBootstrapped<TCreateQueueSchemaActorV2>
-{
-public:
- TCreateQueueSchemaActorV2(const TQueuePath& path,
- const TCreateQueueRequest& req,
+{
+public:
+ TCreateQueueSchemaActorV2(const TQueuePath& path,
+ const TCreateQueueRequest& req,
const TActorId& sender,
- const TString& requestId,
- const TString& customQueueName,
- const TString& folderId,
- const bool isCloudMode,
- const bool enableQueueAttributesValidation,
+ const TString& requestId,
+ const TString& customQueueName,
+ const TString& folderId,
+ const bool isCloudMode,
+ const bool enableQueueAttributesValidation,
TIntrusivePtr<TUserCounters> userCounters,
TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResources);
-
- ~TCreateQueueSchemaActorV2();
-
- void InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config);
-
+
+ ~TCreateQueueSchemaActorV2();
+
+ void InitMissingQueueAttributes(const NKikimrConfig::TSqsConfig& config);
+
void Bootstrap();
-
+
void RequestQueueParams();
-
+
STATEFN(Preamble);
-
+
void HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev);
-
+
void OnReadQueueParams(TSqsEvents::TEvExecuted::TPtr& ev);
-
+
void RunAtomicCounterIncrement();
void OnAtomicCounterIncrement(TSqsEvents::TEvAtomicCounterIncrementResult::TPtr& ev);
-
+
void RequestCreateQueueQuota();
void OnCreateQueueQuota(TEvQuota::TEvClearance::TPtr& ev);
void RequestTablesFormatSettings(const TString& accountName);
void RegisterMakeDirActor(const TString& workingDir, const TString& dirName);
-
+
void RequestLeaderTabletId();
-
+
void CreateComponents();
-
+
STATEFN(CreateComponentsState);
-
+
void Step();
-
+
void OnExecuted(TSqsEvents::TEvExecuted::TPtr& ev);
-
+
void OnDescribeSchemeResult(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev);
-
+
void SendDescribeTable();
void HandleTableDescription(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev);
- template <typename T>
- T GetAttribute(const TStringBuf name, const T& defaultValue) const;
-
+ template <typename T>
+ T GetAttribute(const TStringBuf name, const T& defaultValue) const;
+
void CommitNewVersion();
-
+
STATEFN(FinalizeAndCommit);
-
+
void OnCommit(TSqsEvents::TEvExecuted::TPtr& ev);
-
+
void MatchQueueAttributes(const ui64 currentVersion);
-
+
STATEFN(MatchAttributes);
-
+
void OnAttributesMatch(TSqsEvents::TEvExecuted::TPtr& ev);
-
+
void AddRPSQuota();
void HandleAddQuoterResource(NKesus::TEvKesus::TEvAddQuoterResourceResult::TPtr& ev);
@@ -88,122 +88,122 @@ public:
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::SQS_ACTOR;
- }
-
-private:
- enum class ECreateComponentsStep : ui32 {
+ }
+
+private:
+ enum class ECreateComponentsStep : ui32 {
GetTablesFormatSetting,
MakeQueueDir,
- MakeQueueVersionDir,
- MakeShards,
- MakeTables,
+ MakeQueueVersionDir,
+ MakeShards,
+ MakeTables,
DescribeTableForSetSchemeShardId,
DiscoverLeaderTabletId,
AddQuoterResource,
- };
-
- const TQueuePath QueuePath_;
- const TCreateQueueRequest Request_;
+ };
+
+ const TQueuePath QueuePath_;
+ const TCreateQueueRequest Request_;
const TActorId Sender_;
- const TString CustomQueueName_;
- const TString FolderId_;
- const TString RequestId_;
- const TString GeneratedQueueId_;
- const TInstant QueueCreationTimestamp_;
-
- bool IsFifo_ = false;
- bool IsCloudMode_ = false;
- bool EnableQueueAttributesValidation_ = true;
-
+ const TString CustomQueueName_;
+ const TString FolderId_;
+ const TString RequestId_;
+ const TString GeneratedQueueId_;
+ const TInstant QueueCreationTimestamp_;
+
+ bool IsFifo_ = false;
+ bool IsCloudMode_ = false;
+ bool EnableQueueAttributesValidation_ = true;
+
ui32 TablesFormat_ = 0;
ui64 Version_ = 0;
-
- TString VersionName_;
- TString VersionedQueueFullPath_;
- TString ExistingQueueResourceId_;
+
+ TString VersionName_;
+ TString VersionedQueueFullPath_;
+ TString ExistingQueueResourceId_;
TIntrusivePtr<TUserCounters> UserCounters_;
TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> QuoterResources_;
- ui64 RequiredShardsCount_ = 0;
- ui64 CreatedShardsCount_ = 0;
- TVector<TTable> RequiredTables_;
- ui64 CreatedTablesCount_ = 0;
- TQueueAttributes ValidatedAttributes_;
-
+ ui64 RequiredShardsCount_ = 0;
+ ui64 CreatedShardsCount_ = 0;
+ TVector<TTable> RequiredTables_;
+ ui64 CreatedTablesCount_ = 0;
+ TQueueAttributes ValidatedAttributes_;
+
ui64 LeaderTabletId_ = 0;
TActorId CreateTableWithLeaderTabletActorId_;
ui64 CreateTableWithLeaderTabletTxId_ = 0;
std::pair<ui64, ui64> TableWithLeaderPathId_ = std::make_pair(0, 0); // (scheme shard, path id) are required for describing table
-
+
ECreateComponentsStep CurrentCreationStep_ = ECreateComponentsStep::GetTablesFormatSetting;
TActorId AddQuoterResourceActor_;
-};
-
-class TDeleteQueueSchemaActorV2
+};
+
+class TDeleteQueueSchemaActorV2
: public TActorBootstrapped<TDeleteQueueSchemaActorV2>
-{
-public:
- TDeleteQueueSchemaActorV2(const TQueuePath& path,
+{
+public:
+ TDeleteQueueSchemaActorV2(const TQueuePath& path,
const TActorId& sender,
- const TString& requestId,
+ const TString& requestId,
TIntrusivePtr<TUserCounters> userCounters);
-
- TDeleteQueueSchemaActorV2(const TQueuePath& path,
+
+ TDeleteQueueSchemaActorV2(const TQueuePath& path,
const TActorId& sender,
- const TString& requestId,
+ const TString& requestId,
TIntrusivePtr<TUserCounters> userCounters,
- const ui64 advisedQueueVersion,
- const ui64 advisedShardCount,
- const bool advisedIsFifoFlag);
-
+ const ui64 advisedQueueVersion,
+ const ui64 advisedShardCount,
+ const bool advisedIsFifoFlag);
+
void Bootstrap();
-
+
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::SQS_ACTOR;
- }
-
-private:
- void PrepareCleanupPlan(const bool isFifo, const ui64 shardCount);
-
+ }
+
+private:
+ void PrepareCleanupPlan(const bool isFifo, const ui64 shardCount);
+
void NextAction();
-
+
void DoSuccessOperation();
-
+
void DeleteRPSQuota();
-private:
+private:
STATEFN(StateFunc) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TSqsEvents::TEvExecuted, HandleExecuted);
hFunc(NKesus::TEvKesus::TEvDeleteQuoterResourceResult, HandleDeleteQuoterResource);
cFunc(TEvPoisonPill::EventType, PassAway);
- }
- }
-
+ }
+ }
+
void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev);
void HandleDeleteQuoterResource(NKesus::TEvKesus::TEvDeleteQuoterResourceResult::TPtr& ev);
void PassAway() override;
-
-private:
- enum class EDeleting : ui32 {
- EraseQueueRecord,
- RemoveTables,
- RemoveShards,
- RemoveQueueVersionDirectory,
- RemoveQueueDirectory,
+
+private:
+ enum class EDeleting : ui32 {
+ EraseQueueRecord,
+ RemoveTables,
+ RemoveShards,
+ RemoveQueueVersionDirectory,
+ RemoveQueueDirectory,
DeleteQuoterResource,
- Finish,
- };
-
- const TQueuePath QueuePath_;
+ Finish,
+ };
+
+ const TQueuePath QueuePath_;
const TActorId Sender_;
- TVector<TTable> Tables_;
- TVector<int> Shards_;
- ui32 SI_;
- const TString RequestId_;
+ TVector<TTable> Tables_;
+ TVector<int> Shards_;
+ ui32 SI_;
+ const TString RequestId_;
TIntrusivePtr<TUserCounters> UserCounters_;
- ui64 Version_ = 0;
+ ui64 Version_ = 0;
TActorId DeleteQuoterResourceActor_;
-};
-
+};
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/queues_list_reader.cpp b/ydb/core/ymq/actor/queues_list_reader.cpp
index 865475bd60..37b8b73426 100644
--- a/ydb/core/ymq/actor/queues_list_reader.cpp
+++ b/ydb/core/ymq/actor/queues_list_reader.cpp
@@ -24,8 +24,8 @@ STATEFN(TQueuesListReader::StateFunc) {
}
void TQueuesListReader::HandleReadQueuesList(TSqsEvents::TEvReadQueuesList::TPtr& ev) {
- Recipients.insert(ev->Sender);
-
+ Recipients.insert(ev->Sender);
+
if (!ListingQueues) {
ListingQueues = true;
Result = MakeHolder<TSqsEvents::TEvQueuesList>();
@@ -60,7 +60,7 @@ void TQueuesListReader::OnRequestCompiled(const TSqsEvents::TEvExecuted::TRecord
NextRequest();
} else {
LOG_SQS_WARN("Get queues list request compilation failed: " << record);
- Fail();
+ Fail();
}
}
@@ -126,7 +126,7 @@ void TQueuesListReader::OnQueuesList(const TSqsEvents::TEvExecuted::TRecord& rec
}
rec.FolderId = row["FolderId"];
rec.ShardsCount = row["Shards"];
- rec.DlqName = row["DlqName"];
+ rec.DlqName = row["DlqName"];
rec.CreatedTimestamp = TInstant::MilliSeconds(ui64(row["CreatedTimestamp"]));
}
@@ -137,38 +137,38 @@ void TQueuesListReader::OnQueuesList(const TSqsEvents::TEvExecuted::TRecord& rec
NextRequest();
} else {
std::sort(Result->SortedQueues.begin(), Result->SortedQueues.end()); // If .Queues table consists of many shards, result is possibly not sorted.
- Success();
+ Success();
}
} else {
LOG_SQS_WARN("Get queues list request failed: " << record);
- Fail();
+ Fail();
}
}
-void TQueuesListReader::Success() {
- if (Recipients.size() == 1) {
- Send(*Recipients.begin(), std::move(Result));
- } else {
- for (const auto& recipientId : Recipients) {
- auto result = MakeHolder<TSqsEvents::TEvQueuesList>();
- result->SortedQueues = Result->SortedQueues;
- result->Success = Result->Success;
- Send(recipientId, result.Release());
- }
- }
-
- Recipients.clear();
-
+void TQueuesListReader::Success() {
+ if (Recipients.size() == 1) {
+ Send(*Recipients.begin(), std::move(Result));
+ } else {
+ for (const auto& recipientId : Recipients) {
+ auto result = MakeHolder<TSqsEvents::TEvQueuesList>();
+ result->SortedQueues = Result->SortedQueues;
+ result->Success = Result->Success;
+ Send(recipientId, result.Release());
+ }
+ }
+
+ Recipients.clear();
+
ListingQueues = false;
}
-void TQueuesListReader::Fail() {
- for (const auto& recipientId : Recipients) {
- Send(recipientId, new TSqsEvents::TEvQueuesList(false));
- }
-
- Recipients.clear();
-
+void TQueuesListReader::Fail() {
+ for (const auto& recipientId : Recipients) {
+ Send(recipientId, new TSqsEvents::TEvQueuesList(false));
+ }
+
+ Recipients.clear();
+
ListingQueues = false;
}
diff --git a/ydb/core/ymq/actor/queues_list_reader.h b/ydb/core/ymq/actor/queues_list_reader.h
index a8924be8f8..6562ad4036 100644
--- a/ydb/core/ymq/actor/queues_list_reader.h
+++ b/ydb/core/ymq/actor/queues_list_reader.h
@@ -31,8 +31,8 @@ private:
void NextRequest();
void OnQueuesList(const TSqsEvents::TEvExecuted::TRecord& record);
- void Success();
- void Fail();
+ void Success();
+ void Fail();
private:
TString CompiledQuery;
diff --git a/ydb/core/ymq/actor/receive_message.cpp b/ydb/core/ymq/actor/receive_message.cpp
index 0060168208..2b5bf0e517 100644
--- a/ydb/core/ymq/actor/receive_message.cpp
+++ b/ydb/core/ymq/actor/receive_message.cpp
@@ -31,7 +31,7 @@ public:
{
CopyAccountName(Request());
Response_.MutableReceiveMessage()->SetRequestId(RequestId_);
-
+
CopySecurityToken(Request());
}
@@ -185,8 +185,8 @@ private:
item->SetSentTimestamp(message.SentTimestamp.MilliSeconds());
if (message.SenderId) {
item->SetSenderId(message.SenderId);
- }
-
+ }
+
if (message.MessageAttributes) {
TMessageAttributeList attrs;
if (attrs.ParseFromString(message.MessageAttributes)) {
diff --git a/ydb/core/ymq/actor/schema.cpp b/ydb/core/ymq/actor/schema.cpp
index 6e3f2bbd67..20e84581fe 100644
--- a/ydb/core/ymq/actor/schema.cpp
+++ b/ydb/core/ymq/actor/schema.cpp
@@ -27,26 +27,26 @@ extern const TString RPS_QUOTA_NAME = "RPSQuota";
namespace {
-static const char* const GetNextAtomicValueQuery = R"__(
- (
- (let counterTable '%1$s/.AtomicCounter)
- (let counterRow '(
- '('counter_key (Uint64 '0))))
- (let counterSelect '(
- 'value))
- (let counterRead
- (SelectRow counterTable counterRow counterSelect))
- (let newValue
- (Add (Member counterRead 'value) (Uint64 '1)))
- (let counterUpdate '(
- '('value newValue)))
- (return (Extend
- (AsList (SetResult 'value newValue))
- (AsList (UpdateRow counterTable counterRow counterUpdate))
- ))
- )
-)__";
-
+static const char* const GetNextAtomicValueQuery = R"__(
+ (
+ (let counterTable '%1$s/.AtomicCounter)
+ (let counterRow '(
+ '('counter_key (Uint64 '0))))
+ (let counterSelect '(
+ 'value))
+ (let counterRead
+ (SelectRow counterTable counterRow counterSelect))
+ (let newValue
+ (Add (Member counterRead 'value) (Uint64 '1)))
+ (let counterUpdate '(
+ '('value newValue)))
+ (return (Extend
+ (AsList (SetResult 'value newValue))
+ (AsList (UpdateRow counterTable counterRow counterUpdate))
+ ))
+ )
+)__";
+
static bool SuccessStatusCode(ui32 code) {
switch (NTxProxy::TResultStatus::EStatus(code)) {
case NTxProxy::TResultStatus::EStatus::ExecComplete:
@@ -110,31 +110,31 @@ static void SetOnePartitionPerShardSettings(NKikimrSchemeOp::TTableDescription&
}
}
-} // namespace
-
-THolder<TEvTxUserProxy::TEvProposeTransaction>
- MakeExecuteEvent(const TString& query)
-{
- auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
- trans->SetMode(NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE_AND_EXEC);
- trans->SetFlatMKQL(true);
- trans->MutableProgram()->SetText(query);
-
- return ev;
-}
-
-THolder<TEvTxUserProxy::TEvProposeTransaction>
+} // namespace
+
+THolder<TEvTxUserProxy::TEvProposeTransaction>
+ MakeExecuteEvent(const TString& query)
+{
+ auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
+ trans->SetMode(NKikimrTxUserProxy::TMiniKQLTransaction::COMPILE_AND_EXEC);
+ trans->SetFlatMKQL(true);
+ trans->MutableProgram()->SetText(query);
+
+ return ev;
+}
+
+THolder<TEvTxUserProxy::TEvProposeTransaction>
MakeCreateTableEvent(const TString& root,
const TTable& table,
- size_t queueShardsCount)
+ size_t queueShardsCount)
{
auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
// Transaction info
auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
if (table.Shard != -1) {
- trans->SetWorkingDir(TString::Join(root, "/", ToString(table.Shard)));
+ trans->SetWorkingDir(TString::Join(root, "/", ToString(table.Shard)));
} else {
trans->SetWorkingDir(root);
}
@@ -182,37 +182,37 @@ THolder<TEvTxUserProxy::TEvProposeTransaction>
return ev;
}
-THolder<TEvTxUserProxy::TEvProposeTransaction>
- MakeDeleteTableEvent(const TString& root,
- const TTable& table)
-{
- auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- // Transaction info
- auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
- if (table.Shard == -1) {
- trans->SetWorkingDir(root);
- } else {
- trans->SetWorkingDir(root + "/" + ToString(table.Shard));
- }
+THolder<TEvTxUserProxy::TEvProposeTransaction>
+ MakeDeleteTableEvent(const TString& root,
+ const TTable& table)
+{
+ auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ // Transaction info
+ auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
+ if (table.Shard == -1) {
+ trans->SetWorkingDir(root);
+ } else {
+ trans->SetWorkingDir(root + "/" + ToString(table.Shard));
+ }
trans->SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable);
- trans->MutableDrop()->SetName(table.Name);
-
- return ev;
-}
-
-THolder<TEvTxUserProxy::TEvProposeTransaction>
- MakeRemoveDirectoryEvent(const TString& root, const TString& name)
-{
- auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- // Transaction info
- auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
- trans->SetWorkingDir(root);
+ trans->MutableDrop()->SetName(table.Name);
+
+ return ev;
+}
+
+THolder<TEvTxUserProxy::TEvProposeTransaction>
+ MakeRemoveDirectoryEvent(const TString& root, const TString& name)
+{
+ auto ev = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ // Transaction info
+ auto* trans = ev->Record.MutableTransaction()->MutableModifyScheme();
+ trans->SetWorkingDir(root);
trans->SetOperationType(NKikimrSchemeOp::ESchemeOpRmDir);
- trans->MutableDrop()->SetName(name);
-
- return ev;
-}
-
+ trans->MutableDrop()->SetName(name);
+
+ return ev;
+}
+
THolder<TEvTxUserProxy::TEvProposeTransaction>
MakeCreateKesusEvent(const TString& root,
const TString& kesusName)
@@ -248,12 +248,12 @@ THolder<TEvTxUserProxy::TEvProposeTransaction>
}
TCreateUserSchemaActor::TCreateUserSchemaActor(const TString& root,
- const TString& userName,
+ const TString& userName,
const TActorId& sender,
const TString& requestId,
TIntrusivePtr<TUserCounters> userCounters)
: Root_(root)
- , UserName_(userName)
+ , UserName_(userName)
, Sender_(sender)
, SI_(static_cast<int>(ECreating::MakeDirectory))
, RequestId_(requestId)
@@ -459,36 +459,36 @@ void TDeleteUserSchemaActor::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
}
TAtomicCounterActor::TAtomicCounterActor(const TActorId& sender, const TString& rootPath, const TString& requestId)
- : Sender_(sender)
- , RootPath_(rootPath)
- , RequestId_(requestId)
-{
-}
-
-TAtomicCounterActor::~TAtomicCounterActor() = default;
-
+ : Sender_(sender)
+ , RootPath_(rootPath)
+ , RequestId_(requestId)
+{
+}
+
+TAtomicCounterActor::~TAtomicCounterActor() = default;
+
void TAtomicCounterActor::Bootstrap() {
- Become(&TThis::StateFunc);
- auto ev = MakeExecuteEvent(Sprintf(GetNextAtomicValueQuery, RootPath_.c_str()));
+ Become(&TThis::StateFunc);
+ auto ev = MakeExecuteEvent(Sprintf(GetNextAtomicValueQuery, RootPath_.c_str()));
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), true, TQueuePath(), nullptr));
-}
-
+}
+
void TAtomicCounterActor::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
- const auto& record = ev->Get()->Record;
- const auto status = record.GetStatus();
-
- if (SuccessStatusCode(status)) {
- const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
+ const auto& record = ev->Get()->Record;
+ const auto status = record.GetStatus();
+
+ if (SuccessStatusCode(status)) {
+ const TValue val(TValue::Create(record.GetExecutionEngineEvaluatedResponse()));
Send(Sender_,
- MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(true, "ok", val["value"]));
- } else {
+ MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(true, "ok", val["value"]));
+ } else {
RLOG_SQS_ERROR("Failed to increment the atomic counter: bad status code");
Send(Sender_,
- MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(false));
- }
+ MakeHolder<TSqsEvents::TEvAtomicCounterIncrementResult>(false));
+ }
PassAway();
-}
-
+}
+
template <class TEvCmd, class TEvCmdResult>
class TQuoterCmdRunner : public TActorBootstrapped<TQuoterCmdRunner<TEvCmd, TEvCmdResult>> {
public:
diff --git a/ydb/core/ymq/actor/schema.h b/ydb/core/ymq/actor/schema.h
index 76a5e624c0..c88face921 100644
--- a/ydb/core/ymq/actor/schema.h
+++ b/ydb/core/ymq/actor/schema.h
@@ -21,21 +21,21 @@ namespace NKikimr::NSQS {
extern const TString QUOTER_KESUS_NAME;
extern const TString RPS_QUOTA_NAME;
-THolder<TEvTxUserProxy::TEvProposeTransaction>
- MakeExecuteEvent(const TString& query);
-
-THolder<TEvTxUserProxy::TEvProposeTransaction>
- MakeCreateTableEvent(const TString& root,
- const TTable& table,
- size_t queueShardsCount = 0);
-
-THolder<TEvTxUserProxy::TEvProposeTransaction>
- MakeDeleteTableEvent(const TString& root,
- const TTable& table);
-
-THolder<TEvTxUserProxy::TEvProposeTransaction>
- MakeRemoveDirectoryEvent(const TString& root, const TString& name);
-
+THolder<TEvTxUserProxy::TEvProposeTransaction>
+ MakeExecuteEvent(const TString& query);
+
+THolder<TEvTxUserProxy::TEvProposeTransaction>
+ MakeCreateTableEvent(const TString& root,
+ const TTable& table,
+ size_t queueShardsCount = 0);
+
+THolder<TEvTxUserProxy::TEvProposeTransaction>
+ MakeDeleteTableEvent(const TString& root,
+ const TTable& table);
+
+THolder<TEvTxUserProxy::TEvProposeTransaction>
+ MakeRemoveDirectoryEvent(const TString& root, const TString& name);
+
// Create actor that calls AddQuoterResource and handles pipe errors and retries
TActorId RunAddQuoterResource(ui64 quoterSchemeShardId, ui64 quoterPathId, const NKikimrKesus::TEvAddQuoterResource& cmd, const TString& requestId);
TActorId RunAddQuoterResource(const TString& quoterPath, const NKikimrKesus::TEvAddQuoterResource& cmd, const TString& requestId);
@@ -94,7 +94,7 @@ private:
};
const TString Root_;
- const TString UserName_;
+ const TString UserName_;
const TActorId Sender_;
int SI_;
const TString RequestId_;
@@ -148,32 +148,32 @@ private:
TIntrusivePtr<TUserCounters> UserCounters_;
};
-class TAtomicCounterActor
+class TAtomicCounterActor
: public TActorBootstrapped<TAtomicCounterActor>
-{
-public:
+{
+public:
TAtomicCounterActor(const TActorId& sender, const TString& rootPath, const TString& requestId);
- ~TAtomicCounterActor();
-
+ ~TAtomicCounterActor();
+
void Bootstrap();
-
+
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::SQS_ACTOR;
- }
-
-private:
+ }
+
+private:
STATEFN(StateFunc) {
- switch (ev->GetTypeRewrite()) {
+ switch (ev->GetTypeRewrite()) {
hFunc(TSqsEvents::TEvExecuted, HandleExecuted);
- }
- }
-
+ }
+ }
+
void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev);
-
-private:
+
+private:
const TActorId Sender_;
- const TString RootPath_;
- const TString RequestId_;
-};
-
+ const TString RootPath_;
+ const TString RequestId_;
+};
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/service.cpp b/ydb/core/ymq/actor/service.cpp
index caf26a509e..34c9a4ae46 100644
--- a/ydb/core/ymq/actor/service.cpp
+++ b/ydb/core/ymq/actor/service.cpp
@@ -1,9 +1,9 @@
#include "service.h"
-
+
#include "auth_factory.h"
#include "cfg.h"
#include "executor.h"
-#include "garbage_collector.h"
+#include "garbage_collector.h"
#include "local_rate_limiter_allocator.h"
#include "params.h"
#include "proxy_service.h"
@@ -78,8 +78,8 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> {
, QueueName_(std::move(queueName))
, CustomName_(std::move(customName))
, FolderId_(std::move(folderId))
- , Version_(version)
- , ShardsCount_(shardsCount)
+ , Version_(version)
+ , ShardsCount_(shardsCount)
, RootUrl_(std::move(rootUrl))
, LeaderTabletId_(leaderTabletId)
, Counters_(userCounters->CreateQueueCounters(QueueName_, FolderId_, insertCounters))
@@ -182,8 +182,8 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> {
TString QueueName_;
TString CustomName_;
TString FolderId_;
- ui64 Version_;
- ui64 ShardsCount_;
+ ui64 Version_;
+ ui64 ShardsCount_;
TString RootUrl_;
ui64 LeaderTabletId_ = 0;
TIntrusivePtr<TQueueCounters> Counters_;
@@ -239,14 +239,14 @@ struct TSqsService::TUserInfo : public TAtomicRefCount<TUserInfo> {
}
}
- size_t CountQueuesInFolder(const TString& folderId) const {
- if (!folderId) {
- return QueueByNameAndFolder_.size(); // for YaSQS
- }
-
- return std::count_if(QueueByNameAndFolder_.begin(), QueueByNameAndFolder_.end(), [&folderId](const auto& p) { return p.first.second == folderId; });
- }
-
+ size_t CountQueuesInFolder(const TString& folderId) const {
+ if (!folderId) {
+ return QueueByNameAndFolder_.size(); // for YaSQS
+ }
+
+ return std::count_if(QueueByNameAndFolder_.begin(), QueueByNameAndFolder_.end(), [&folderId](const auto& p) { return p.first.second == folderId; });
+ }
+
TString UserName_;
std::shared_ptr<const std::map<TString, TString>> Settings_ = std::make_shared<const std::map<TString, TString>>();
TIntrusivePtr<TUserCounters> Counters_;
@@ -262,8 +262,8 @@ struct TSqsService::TUserInfo : public TAtomicRefCount<TUserInfo> {
THashMultiMap<TString, TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr> GetLeaderNodeRequests_; // queue name -> request
THashMultiMap<TString, TSqsEvents::TEvGetConfiguration::TPtr> GetConfigurationRequests_; // queue name -> request
THashMultiMap<std::pair<TString, TString>, TSqsEvents::TEvGetQueueId::TPtr> GetQueueIdRequests_; // <queue custom name, folder id> -> request
- THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // queue name -> request
- THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // folder id -> request
+ THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // queue name -> request
+ THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // folder id -> request
};
static TString GetEndpoint(const NKikimrConfig::TSqsConfig& config) {
@@ -310,8 +310,8 @@ void TSqsService::Bootstrap() {
Register(new TUserSettingsReader(AggregatedUserCounters_->GetTransactionCounters()));
QueuesListReader_ = Register(new TQueuesListReader(AggregatedUserCounters_->GetTransactionCounters()));
- Register(CreateGarbageCollector(SchemeCache_, QueuesListReader_));
-
+ Register(CreateGarbageCollector(SchemeCache_, QueuesListReader_));
+
RequestSqsUsersList();
RequestSqsQueuesList();
@@ -541,8 +541,8 @@ void TSqsService::AnswerNotExists(TSqsEvents::TEvGetQueueId::TPtr& ev, const TUs
void TSqsService::AnswerNotExists(TSqsEvents::TEvCountQueues::TPtr& ev, const TUserInfoPtr&) {
RLOG_SQS_REQ_DEBUG(ev->Get()->RequestId, "No user [" << ev->Get()->UserName << "] found while counting queues");
Send(ev->Sender, new TSqsEvents::TEvCountQueuesResponse(false));
-}
-
+}
+
void TSqsService::AnswerFailed(TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr& ev, const TUserInfoPtr&) {
Send(ev->Sender, new TSqsEvents::TEvGetLeaderNodeForQueueResponse(ev->Get()->RequestId, ev->Get()->UserName, ev->Get()->QueueName, TSqsEvents::TEvGetLeaderNodeForQueueResponse::EStatus::Error));
}
@@ -568,8 +568,8 @@ void TSqsService::AnswerFailed(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPt
void TSqsService::AnswerFailed(TSqsEvents::TEvCountQueues::TPtr& ev, const TUserInfoPtr&) {
Send(ev->Sender, new TSqsEvents::TEvCountQueuesResponse(true));
-}
-
+}
+
void TSqsService::Answer(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr& ev, const TQueueInfoPtr& queueInfo) {
Send(ev->Sender, new TSqsEvents::TEvQueueFolderIdAndCustomName(queueInfo->FolderId_, queueInfo->CustomName_));
}
@@ -699,13 +699,13 @@ void TSqsService::HandleGetQueueFolderIdAndCustomName(TSqsEvents::TEvGetQueueFol
void TSqsService::HandleCountQueues(TSqsEvents::TEvCountQueues::TPtr& ev) {
TUserInfoPtr user = GetUserOrWait(ev);
- if (!user) {
- return;
- }
-
+ if (!user) {
+ return;
+ }
+
Send(ev->Sender, new TSqsEvents::TEvCountQueuesResponse(false, true, user->CountQueuesInFolder(ev->Get()->FolderId)));
-}
-
+}
+
template <class TEvent>
TSqsService::TUserInfoPtr TSqsService::GetUserOrWait(TAutoPtr<TEvent>& ev) {
const TString& reqId = ev->Get()->RequestId;
@@ -861,7 +861,7 @@ void TSqsService::HandleQueuesList(TSqsEvents::TEvQueuesList::TPtr& ev) {
++usersIt;
}
}
-
+
NotifyLocalDeadLetterQueuesLeaders(ev->Get()->SortedQueues);
} else {
for (const auto& [userName, user] : Users_) {
@@ -869,45 +869,45 @@ void TSqsService::HandleQueuesList(TSqsEvents::TEvQueuesList::TPtr& ev) {
}
}
}
-
+
void TSqsService::NotifyLocalDeadLetterQueuesLeaders(const std::vector<TSqsEvents::TEvQueuesList::TQueueRecord>& sortedQueues) const {
- using TKnownDeadLetterQueues = THashMap<TString, THashSet<std::pair<TString, TString>>>;
-
- TKnownDeadLetterQueues knownDlqs;
- for (const auto& queueRecord : sortedQueues) {
- if (queueRecord.DlqName) {
- knownDlqs[queueRecord.UserName].insert(std::make_pair(queueRecord.DlqName, queueRecord.FolderId)); // account -> custom name + folder id pair
- }
- }
-
- for (const auto& [account, dlqs] : knownDlqs) {
- auto accountIt = Users_.find(account);
- if (accountIt != Users_.end()) {
- for (const auto& customNameAndFolderPair : dlqs) {
- auto queueInfoIt = accountIt->second->QueueByNameAndFolder_.find(customNameAndFolderPair);
- if (queueInfoIt != accountIt->second->QueueByNameAndFolder_.end()) {
- const auto& queueInfo = *queueInfoIt->second;
+ using TKnownDeadLetterQueues = THashMap<TString, THashSet<std::pair<TString, TString>>>;
+
+ TKnownDeadLetterQueues knownDlqs;
+ for (const auto& queueRecord : sortedQueues) {
+ if (queueRecord.DlqName) {
+ knownDlqs[queueRecord.UserName].insert(std::make_pair(queueRecord.DlqName, queueRecord.FolderId)); // account -> custom name + folder id pair
+ }
+ }
+
+ for (const auto& [account, dlqs] : knownDlqs) {
+ auto accountIt = Users_.find(account);
+ if (accountIt != Users_.end()) {
+ for (const auto& customNameAndFolderPair : dlqs) {
+ auto queueInfoIt = accountIt->second->QueueByNameAndFolder_.find(customNameAndFolderPair);
+ if (queueInfoIt != accountIt->second->QueueByNameAndFolder_.end()) {
+ const auto& queueInfo = *queueInfoIt->second;
if (queueInfo.LocalLeader_) {
Send(queueInfo.LocalLeader_, new TSqsEvents::TEvDeadLetterQueueNotification);
- }
- }
- }
- }
- }
-}
-
+ }
+ }
+ }
+ }
+ }
+}
+
void TSqsService::AnswerCountQueuesRequests(const TUserInfoPtr& user) {
while (!user->CountQueuesRequests_.empty()) {
const TString folderId = user->CountQueuesRequests_.begin()->first;
const auto queuesCount = user->CountQueuesInFolder(folderId);
-
+
auto requests = user->CountQueuesRequests_.equal_range(folderId);
-
+
for (auto i = requests.first; i != requests.second; ++i) {
auto& req = i->second;
Send(req->Sender, new TSqsEvents::TEvCountQueuesResponse(false, true, queuesCount));
- }
-
+ }
+
user->CountQueuesRequests_.erase(requests.first, requests.second);
}
}
@@ -1034,8 +1034,8 @@ std::map<TString, TSqsService::TQueueInfoPtr>::iterator TSqsService::AddQueue(co
ui64 leaderTabletId,
const TString& customName,
const TString& folderId,
- const ui64 version,
- const ui64 shardsCount,
+ const ui64 version,
+ const ui64 shardsCount,
const TInstant createdTimestamp) {
auto user = MutableUser(userName, false); // don't move requests because they are already moved in our caller
const TInstant now = TActivationContext::Now();
@@ -1079,12 +1079,12 @@ std::map<TString, TSqsService::TQueueInfoPtr>::iterator TSqsService::AddQueue(co
}
{
- auto requests = user->GetQueueFolderIdAndCustomNameRequests_.equal_range(queue);
+ auto requests = user->GetQueueFolderIdAndCustomNameRequests_.equal_range(queue);
for (auto i = requests.first; i != requests.second; ++i) {
auto& req = i->second;
Answer(req, queueInfo);
}
- user->GetQueueFolderIdAndCustomNameRequests_.erase(requests.first, requests.second);
+ user->GetQueueFolderIdAndCustomNameRequests_.erase(requests.first, requests.second);
}
queueInfo->ConnectToLeaderTablet();
@@ -1209,8 +1209,8 @@ void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev) {
GetQueueIdRequests_.emplace(ev->Get()->UserName, std::move(ev));
}
-void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev) {
- GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->UserName, std::move(ev));
+void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev) {
+ GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->UserName, std::move(ev));
}
void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev) {
@@ -1221,16 +1221,16 @@ void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueReque
GetLeaderNodeRequests_.emplace(ev->Get()->UserName, std::move(ev));
}
-void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev) {
- CountQueuesRequests_.emplace(ev->Get()->UserName, std::move(ev));
-}
-
+void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev) {
+ CountQueuesRequests_.emplace(ev->Get()->UserName, std::move(ev));
+}
+
void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev, const TUserInfoPtr& userInfo) {
userInfo->GetQueueIdRequests_.emplace(std::make_pair(ev->Get()->CustomQueueName, ev->Get()->FolderId), std::move(ev));
}
-void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo) {
- userInfo->GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->QueueName, std::move(ev));
+void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo) {
+ userInfo->GetQueueFolderIdAndCustomNameRequests_.emplace(ev->Get()->QueueName, std::move(ev));
}
void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev, const TUserInfoPtr& userInfo) {
@@ -1241,10 +1241,10 @@ void TSqsService::InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueReque
userInfo->GetLeaderNodeRequests_.emplace(ev->Get()->QueueName, std::move(ev));
}
-void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo) {
- userInfo->CountQueuesRequests_.emplace(ev->Get()->FolderId, std::move(ev));
-}
-
+void TSqsService::InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo) {
+ userInfo->CountQueuesRequests_.emplace(ev->Get()->FolderId, std::move(ev));
+}
+
template <class TMultimap>
size_t TSqsService::MoveUserRequests(const TUserInfoPtr& userInfo, TMultimap& map) {
size_t moved = 0;
diff --git a/ydb/core/ymq/actor/service.h b/ydb/core/ymq/actor/service.h
index 5de1b7f2f0..caf091c24b 100644
--- a/ydb/core/ymq/actor/service.h
+++ b/ydb/core/ymq/actor/service.h
@@ -72,7 +72,7 @@ private:
TUserInfoPtr MutableUser(const TString& userName, bool moveUserRequestsToUserRecord = true, bool* requestsWereMoved = nullptr);
void RemoveUser(const TString& userName);
std::map<TString, TQueueInfoPtr>::iterator AddQueue(const TString& userName, const TString& queue, ui64 leaderTabletId,
- const TString& customName, const TString& folderId, const ui64 version,
+ const TString& customName, const TString& folderId, const ui64 version,
const ui64 shardsCount, const TInstant createdTimestamp);
void AnswerNoUserToRequests();
@@ -91,16 +91,16 @@ private:
TUserInfoPtr GetUserOrWait(TAutoPtr<TEvent>& ev);
void InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev);
- void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev);
+ void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev);
void InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev);
void InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr&& ev);
- void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev);
+ void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev);
void InsertWaitingRequest(TSqsEvents::TEvGetQueueId::TPtr&& ev, const TUserInfoPtr& userInfo);
- void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo);
+ void InsertWaitingRequest(TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr&& ev, const TUserInfoPtr& userInfo);
void InsertWaitingRequest(TSqsEvents::TEvGetConfiguration::TPtr&& ev, const TUserInfoPtr& userInfo);
void InsertWaitingRequest(TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr&& ev, const TUserInfoPtr& userInfo);
- void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo);
+ void InsertWaitingRequest(TSqsEvents::TEvCountQueues::TPtr&& ev, const TUserInfoPtr& userInfo);
template <class TMultimap>
size_t MoveUserRequests(const TUserInfoPtr& userInfo, TMultimap& map); // returns moved requests count
@@ -128,7 +128,7 @@ private:
void AnswerCountQueuesRequests(const TUserInfoPtr& user);
void NotifyLocalDeadLetterQueuesLeaders(const std::vector<TSqsEvents::TEvQueuesList::TQueueRecord>& sortedQueues) const;
-
+
void MakeAndRegisterYcEventsProcessor();
private:
@@ -156,8 +156,8 @@ private:
THashMultiMap<TString, TSqsEvents::TEvGetLeaderNodeForQueueRequest::TPtr> GetLeaderNodeRequests_; // user name -> request
THashMultiMap<TString, TSqsEvents::TEvGetConfiguration::TPtr> GetConfigurationRequests_; // user name -> request
THashMultiMap<TString, TSqsEvents::TEvGetQueueId::TPtr> GetQueueIdRequests_; // user name -> request
- THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // user name -> request
- THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // user name -> request
+ THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // user name -> request
+ THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // user name -> request
struct TYcSearchEventsConfig {
diff --git a/ydb/core/ymq/actor/serviceid.h b/ydb/core/ymq/actor/serviceid.h
index a757bb69c1..7219d918ba 100644
--- a/ydb/core/ymq/actor/serviceid.h
+++ b/ydb/core/ymq/actor/serviceid.h
@@ -4,7 +4,7 @@
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/logger/file.h>
-
+
#include <util/generic/strbuf.h>
namespace NKikimr::NSQS {
@@ -21,16 +21,16 @@ inline TActorId MakeSqsProxyServiceID(ui32 nodeId) {
inline TActorId MakeSqsAccessServiceID() {
return TActorId(0, TStringBuf("SQS_ACCESS"));
-}
-
+}
+
inline TActorId MakeSqsFolderServiceID() {
return TActorId(0, TStringBuf("SQS_FOLDER"));
-}
-
+}
+
inline TActorId MakeSqsMeteringServiceID() {
return TActorId(0, TStringBuf("SQS_METER"));
-}
-
+}
+
IActor* CreateSqsService(TMaybe<ui32> ydbPort = Nothing());
IActor* CreateSqsProxyService();
IActor* CreateSqsAccessService(const TString& address, const TString& pathToRootCA);
diff --git a/ydb/core/ymq/actor/set_queue_attributes.cpp b/ydb/core/ymq/actor/set_queue_attributes.cpp
index b3e0f8a8a1..df33ad99e6 100644
--- a/ydb/core/ymq/actor/set_queue_attributes.cpp
+++ b/ydb/core/ymq/actor/set_queue_attributes.cpp
@@ -11,7 +11,7 @@
#include <ydb/public/lib/value/value.h>
#include <library/cpp/scheme/scheme.h>
-
+
#include <util/generic/maybe.h>
#include <util/generic/utility.h>
#include <util/string/cast.h>
@@ -33,7 +33,7 @@ public:
for (const auto& attr : Request().attributes()) {
Attributes_[attr.GetName()] = attr.GetValue();
}
-
+
CopySecurityToken(Request());
}
@@ -53,9 +53,9 @@ private:
const bool clampValues = !Cfg().GetEnableQueueAttributesValidation();
ValidatedAttributes_ = TQueueAttributes::FromAttributesAndConfig(Attributes_, Cfg(), IsFifoQueue(), clampValues);
- if (!ValidatedAttributes_.Validate()) {
- MakeError(Response_.MutableSetQueueAttributes(), *ValidatedAttributes_.Error, ValidatedAttributes_.ErrorText);
- return false;
+ if (!ValidatedAttributes_.Validate()) {
+ MakeError(Response_.MutableSetQueueAttributes(), *ValidatedAttributes_.Error, ValidatedAttributes_.ErrorText);
+ return false;
}
return true;
@@ -75,18 +75,18 @@ private:
.Counters(QueueCounters_)
.RetryOnTimeout();
- builder.Params().OptionalUint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount);
- builder.Params().OptionalUtf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn);
- builder.Params().OptionalUtf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName);
-
- builder.Params().OptionalUint64("VISIBILITY", ToMilliSeconds(ValidatedAttributes_.VisibilityTimeout));
- builder.Params().OptionalUint64("DELAY", ToMilliSeconds(ValidatedAttributes_.DelaySeconds));
- builder.Params().OptionalUint64("RETENTION", ToMilliSeconds(ValidatedAttributes_.MessageRetentionPeriod));
- builder.Params().OptionalUint64("WAIT", ToMilliSeconds(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds));
- builder.Params().OptionalUint64("MAX_MESSAGE_SIZE", ValidatedAttributes_.MaximumMessageSize);
+ builder.Params().OptionalUint64("MAX_RECEIVE_COUNT", ValidatedAttributes_.RedrivePolicy.MaxReceiveCount);
+ builder.Params().OptionalUtf8("DLQ_TARGET_ARN", ValidatedAttributes_.RedrivePolicy.TargetArn);
+ builder.Params().OptionalUtf8("DLQ_TARGET_NAME", ValidatedAttributes_.RedrivePolicy.TargetQueueName);
+
+ builder.Params().OptionalUint64("VISIBILITY", ToMilliSeconds(ValidatedAttributes_.VisibilityTimeout));
+ builder.Params().OptionalUint64("DELAY", ToMilliSeconds(ValidatedAttributes_.DelaySeconds));
+ builder.Params().OptionalUint64("RETENTION", ToMilliSeconds(ValidatedAttributes_.MessageRetentionPeriod));
+ builder.Params().OptionalUint64("WAIT", ToMilliSeconds(ValidatedAttributes_.ReceiveMessageWaitTimeSeconds));
+ builder.Params().OptionalUint64("MAX_MESSAGE_SIZE", ValidatedAttributes_.MaximumMessageSize);
if (IsFifoQueue()) {
- builder.Params().OptionalBool("CONTENT_BASED_DEDUPLICATION", ValidatedAttributes_.ContentBasedDeduplication);
+ builder.Params().OptionalBool("CONTENT_BASED_DEDUPLICATION", ValidatedAttributes_.ContentBasedDeduplication);
}
builder.Params().Utf8("USER_NAME", UserName_);
@@ -95,19 +95,19 @@ private:
}
void DoAction() override {
- Become(&TThis::StateFunc);
-
- if (ValidatedAttributes_.HasClampedAttributes()) {
+ Become(&TThis::StateFunc);
+
+ if (ValidatedAttributes_.HasClampedAttributes()) {
RLOG_SQS_WARN("Clamped some queue attribute values for account " << UserName_ << " and queue name " << GetQueueName());
- }
-
- if (ValidatedAttributes_.RedrivePolicy.TargetQueueName && *ValidatedAttributes_.RedrivePolicy.TargetQueueName) {
+ }
+
+ if (ValidatedAttributes_.RedrivePolicy.TargetQueueName && *ValidatedAttributes_.RedrivePolicy.TargetQueueName) {
Send(MakeSqsServiceID(SelfId().NodeId()), new TSqsEvents::TEvGetQueueId(RequestId_, UserName_, *ValidatedAttributes_.RedrivePolicy.TargetQueueName, FolderId_));
- } else {
+ } else {
RequestAttributesChange();
- }
- }
-
+ }
+ }
+
TString DoGetQueueName() const override {
return Request().GetQueueName();
}
@@ -121,21 +121,21 @@ private:
}
void HandleQueueId(TSqsEvents::TEvQueueId::TPtr& ev) {
- if (ev->Get()->Failed) {
+ if (ev->Get()->Failed) {
RLOG_SQS_WARN("Get queue id failed");
- MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
- } else if (!ev->Get()->Exists) {
+ MakeError(MutableErrorDesc(), NErrors::INTERNAL_FAILURE);
+ } else if (!ev->Get()->Exists) {
MakeError(MutableErrorDesc(), NErrors::NON_EXISTENT_QUEUE, "Target DLQ does not exist.");
- } else if (ev->Get()->QueueId == GetQueueName()) {
- MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, "Using the queue itself as a dead letter queue is not allowed.");
- } else {
+ } else if (ev->Get()->QueueId == GetQueueName()) {
+ MakeError(MutableErrorDesc(), NErrors::INVALID_PARAMETER_VALUE, "Using the queue itself as a dead letter queue is not allowed.");
+ } else {
RequestAttributesChange();
- return;
- }
-
+ return;
+ }
+
SendReplyAndDie();
- }
-
+ }
+
void HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
const auto& record = ev->Get()->Record;
const ui32 status = record.GetStatus();
@@ -159,8 +159,8 @@ private:
private:
THashMap<TString, TString> Attributes_;
-
- TQueueAttributes ValidatedAttributes_;
+
+ TQueueAttributes ValidatedAttributes_;
};
IActor* CreateSetQueueAttributesActor(const NKikimrClient::TSqsRequest& sourceSqsRequest, THolder<IReplyCallback> cb) {
diff --git a/ydb/core/ymq/actor/ut/infly_ut.cpp b/ydb/core/ymq/actor/ut/infly_ut.cpp
index bab1c4ae99..f6fed85aff 100644
--- a/ydb/core/ymq/actor/ut/infly_ut.cpp
+++ b/ydb/core/ymq/actor/ut/infly_ut.cpp
@@ -8,20 +8,20 @@ Y_UNIT_TEST_SUITE(InflyTest) {
Y_UNIT_TEST(AddMessage) {
TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>();
UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(42)), 0);
- infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0));
+ infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0));
UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 1);
UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(42)), 1);
UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(43)), 0);
- infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0));
+ infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0));
UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 2);
UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(1)), 2);
}
Y_UNIT_TEST(DeleteMessage) {
TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>();
- infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0));
- infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0));
+ infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0));
+ infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0));
UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 2);
UNIT_ASSERT(!infly->Delete(5));
UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 2);
@@ -35,9 +35,9 @@ Y_UNIT_TEST_SUITE(InflyTest) {
Y_UNIT_TEST(ChangeMesageVisibility) {
TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>();
- infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0));
- infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0));
- infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(150), 0));
+ infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(42), 0));
+ infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(12), 0));
+ infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(150), 0));
UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 3);
UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(50)), 1);
@@ -82,11 +82,11 @@ Y_UNIT_TEST_SUITE(InflyTest) {
Y_UNIT_TEST(ReceiveMessages) {
TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>();
- infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0));
- infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0));
- infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0));
- infly->Add(MakeHolder<TInflyMessage>(4ull, 0ull, TInstant::Seconds(4), 0));
- infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(5), 0));
+ infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0));
+ infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0));
+ infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0));
+ infly->Add(MakeHolder<TInflyMessage>(4ull, 0ull, TInstant::Seconds(4), 0));
+ infly->Add(MakeHolder<TInflyMessage>(5ull, 0ull, TInstant::Seconds(5), 0));
UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 5);
UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(50)), 0);
{
@@ -123,9 +123,9 @@ Y_UNIT_TEST_SUITE(InflyTest) {
Y_UNIT_TEST(DeleteReceivedMessage) {
TIntrusivePtr<TInflyMessages> infly = MakeIntrusive<TInflyMessages>();
- infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0));
- infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0));
- infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0));
+ infly->Add(MakeHolder<TInflyMessage>(1ull, 0ull, TInstant::Seconds(1), 0));
+ infly->Add(MakeHolder<TInflyMessage>(2ull, 0ull, TInstant::Seconds(2), 0));
+ infly->Add(MakeHolder<TInflyMessage>(3ull, 0ull, TInstant::Seconds(3), 0));
UNIT_ASSERT_VALUES_EQUAL(infly->GetCapacity(), 3);
UNIT_ASSERT_VALUES_EQUAL(infly->GetInflyCount(TInstant::Seconds(2)), 2);
{
diff --git a/ydb/core/ymq/actor/ya.make b/ydb/core/ymq/actor/ya.make
index 3599c3756a..070653bb57 100644
--- a/ydb/core/ymq/actor/ya.make
+++ b/ydb/core/ymq/actor/ya.make
@@ -11,7 +11,7 @@ SRCS(
attributes_md5.cpp
cfg.cpp
change_visibility.cpp
- count_queues.cpp
+ count_queues.cpp
create_queue.cpp
create_user.cpp
delete_message.cpp
@@ -20,21 +20,21 @@ SRCS(
error.cpp
executor.cpp
fifo_cleanup.cpp
- garbage_collector.cpp
+ garbage_collector.cpp
get_queue_attributes.cpp
get_queue_url.cpp
index_events_processor.cpp
infly.cpp
log.cpp
- list_dead_letter_source_queues.cpp
- list_permissions.cpp
+ list_dead_letter_source_queues.cpp
+ list_permissions.cpp
list_queues.cpp
list_users.cpp
local_rate_limiter_allocator.cpp
message_delay_stats.cpp
- metering.cpp
+ metering.cpp
migration.cpp
- modify_permissions.cpp
+ modify_permissions.cpp
proxy_actor.cpp
purge.cpp
purge_queue.cpp
@@ -48,14 +48,14 @@ SRCS(
set_queue_attributes.cpp
proxy_service.cpp
queues_list_reader.cpp
- queue_schema.cpp
+ queue_schema.cpp
user_settings_names.cpp
user_settings_reader.cpp
)
PEERDIR(
contrib/libs/openssl
- contrib/libs/protobuf
+ contrib/libs/protobuf
library/cpp/actors/core
library/cpp/containers/intrusive_rb_tree
library/cpp/digest/md5
@@ -96,6 +96,6 @@ YQL_LAST_ABI_VERSION()
GENERATE_ENUM_SERIALIZATION(events.h)
-GENERATE_ENUM_SERIALIZATION(metering.h)
+GENERATE_ENUM_SERIALIZATION(metering.h)
END()
diff --git a/ydb/core/ymq/base/acl.cpp b/ydb/core/ymq/base/acl.cpp
index 0a088aebea..683611e88c 100644
--- a/ydb/core/ymq/base/acl.cpp
+++ b/ydb/core/ymq/base/acl.cpp
@@ -1,141 +1,141 @@
-#include "acl.h"
-
-#include <util/generic/hash.h>
-#include <util/generic/singleton.h>
-#include <util/generic/string.h>
-
-#include <map>
-
+#include "acl.h"
+
+#include <util/generic/hash.h>
+#include <util/generic/singleton.h>
+#include <util/generic/string.h>
+
+#include <map>
+
namespace NKikimr::NSQS {
-class TSQSACLMappings {
-public:
- TSQSACLMappings() {
- static const TStringBuf createQueue = "CreateQueue";
- static const TStringBuf createUser = "CreateUser";
- static const TStringBuf sendMessage = "SendMessage";
- static const TStringBuf receiveMessage = "ReceiveMessage";
- static const TStringBuf deleteMessage = "DeleteMessage";
- static const TStringBuf modifyPermissions = "ModifyPermissions";
- static const TStringBuf alterQueue = "AlterQueue";
- static const TStringBuf describePath = "DescribePath";
- static const TStringBuf deleteUser = "DeleteUser";
- static const TStringBuf deleteQueue = "DeleteQueue";
-
- ACE2AccessIndex = {
- {createQueue, NACLib::EAccessRights::CreateQueue},
- {createUser, NACLib::EAccessRights::CreateTable},
- {sendMessage, NACLib::EAccessRights::UpdateRow},
- {receiveMessage, NACLib::EAccessRights::SelectRow},
- {deleteMessage, NACLib::EAccessRights::EraseRow},
- {modifyPermissions, NACLib::EAccessRights::WriteAttributes},
- {alterQueue, NACLib::EAccessRights::AlterSchema},
- {describePath, NACLib::EAccessRights::DescribeSchema},
- {deleteUser, NACLib::EAccessRights::RemoveSchema},
- {deleteQueue, NACLib::EAccessRights::ReadAttributes}}; // as intended
-
- Action2ACEIndex = {
- {"CreateQueue", {EACLSourceType::AccountDir, createQueue}},
- {"CreateUser", {EACLSourceType::RootDir, createUser}},
- {"SendMessage", {EACLSourceType::QueueDir, sendMessage}},
- {"SendMessageBatch", {EACLSourceType::QueueDir, sendMessage}},
- {"ChangeMessageVisibility", {EACLSourceType::QueueDir, receiveMessage}},
- {"ChangeMessageVisibilityBatch", {EACLSourceType::QueueDir, receiveMessage}},
- {"ReceiveMessage", {EACLSourceType::QueueDir, receiveMessage}},
- {"DeleteMessage", {EACLSourceType::QueueDir, deleteMessage}},
- {"DeleteMessageBatch", {EACLSourceType::QueueDir, deleteMessage}},
- {"PurgeQueue", {EACLSourceType::QueueDir, deleteMessage}},
- {"ModifyPermissions", {EACLSourceType::Custom, modifyPermissions}},
- {"SetQueueAttributes", {EACLSourceType::QueueDir, alterQueue}},
- {"TagQueue", {EACLSourceType::QueueDir, alterQueue}},
- {"UntagQueue", {EACLSourceType::QueueDir, alterQueue}},
- {"GetQueueAttributes", {EACLSourceType::QueueDir, describePath}},
- {"ListQueueTags", {EACLSourceType::QueueDir, describePath}},
- {"GetQueueUrl", {EACLSourceType::QueueDir, describePath}},
- {"ListQueues", {EACLSourceType::AccountDir, describePath}},
- {"ListDeadLetterSourceQueues", {EACLSourceType::QueueDir, describePath}},
- {"ListPermissions", {EACLSourceType::Custom, describePath}},
- {"CountQueues", {EACLSourceType::AccountDir, describePath}},
- {"DeleteUser", {EACLSourceType::AccountDir, deleteUser}},
- {"DeleteQueue", {EACLSourceType::QueueDir, deleteQueue}}};
-
- for (const auto& pair : ACE2AccessIndex) {
- Access2ACEIndex[pair.second] = pair.first;
- }
-
- Y_VERIFY(Access2ACEIndex.size() == ACE2AccessIndex.size());
- }
-
- EACLSourceType GetActionACLSourceTypeImpl(const TStringBuf action) const {
- auto it = Action2ACEIndex.find(action);
- if (it != Action2ACEIndex.end()) {
- return it->second.first;
- }
-
- return EACLSourceType::Unknown;
- }
-
- ui32 GetActionRequiredAccessImpl(const TStringBuf action) const {
- return GetACERequiredAccessImpl(GetActionMatchingACEImpl(action));
- }
-
- ui32 GetACERequiredAccessImpl(const TStringBuf aceName) const {
- auto accessIt = ACE2AccessIndex.find(aceName);
- if (accessIt != ACE2AccessIndex.end()) {
- return accessIt->second;
- }
-
- return 0;
- }
-
- TString GetActionMatchingACEImpl(const TStringBuf actionName) const {
- auto it = Action2ACEIndex.find(actionName);
- if (it != Action2ACEIndex.end()) {
- return TString(it->second.second);
- }
-
- return {};
- }
-
- TVector<TStringBuf> GetAccessMatchingACEImpl(const ui32 access) const {
- TVector<TStringBuf> result;
- for (const auto& pair : Access2ACEIndex) {
- if (pair.first & access) {
- result.push_back(pair.second);
- }
- }
-
- return result;
- }
-
-private:
- THashMap<TStringBuf, ui32> ACE2AccessIndex;
- THashMap<ui32, TStringBuf> Access2ACEIndex;
- THashMap<TStringBuf, std::pair<EACLSourceType, TStringBuf>> Action2ACEIndex;
-};
-
-static const TSQSACLMappings& Mappings() {
- return *Singleton<TSQSACLMappings>();
-}
-
-EACLSourceType GetActionACLSourceType(const TString& actionName) {
- return Mappings().GetActionACLSourceTypeImpl(actionName);
-}
-
-ui32 GetActionRequiredAccess(const TString& actionName) {
- return Mappings().GetActionRequiredAccessImpl(actionName);
-}
-
-ui32 GetACERequiredAccess(const TString& aceName) {
- return Mappings().GetACERequiredAccessImpl(aceName);
-}
-
-TString GetActionMatchingACE(const TString& actionName) {
- return Mappings().GetActionMatchingACEImpl(actionName);
-}
-
-TVector<TStringBuf> GetAccessMatchingACE(const ui32 access) {
- return Mappings().GetAccessMatchingACEImpl(access);
-}
+class TSQSACLMappings {
+public:
+ TSQSACLMappings() {
+ static const TStringBuf createQueue = "CreateQueue";
+ static const TStringBuf createUser = "CreateUser";
+ static const TStringBuf sendMessage = "SendMessage";
+ static const TStringBuf receiveMessage = "ReceiveMessage";
+ static const TStringBuf deleteMessage = "DeleteMessage";
+ static const TStringBuf modifyPermissions = "ModifyPermissions";
+ static const TStringBuf alterQueue = "AlterQueue";
+ static const TStringBuf describePath = "DescribePath";
+ static const TStringBuf deleteUser = "DeleteUser";
+ static const TStringBuf deleteQueue = "DeleteQueue";
+
+ ACE2AccessIndex = {
+ {createQueue, NACLib::EAccessRights::CreateQueue},
+ {createUser, NACLib::EAccessRights::CreateTable},
+ {sendMessage, NACLib::EAccessRights::UpdateRow},
+ {receiveMessage, NACLib::EAccessRights::SelectRow},
+ {deleteMessage, NACLib::EAccessRights::EraseRow},
+ {modifyPermissions, NACLib::EAccessRights::WriteAttributes},
+ {alterQueue, NACLib::EAccessRights::AlterSchema},
+ {describePath, NACLib::EAccessRights::DescribeSchema},
+ {deleteUser, NACLib::EAccessRights::RemoveSchema},
+ {deleteQueue, NACLib::EAccessRights::ReadAttributes}}; // as intended
+
+ Action2ACEIndex = {
+ {"CreateQueue", {EACLSourceType::AccountDir, createQueue}},
+ {"CreateUser", {EACLSourceType::RootDir, createUser}},
+ {"SendMessage", {EACLSourceType::QueueDir, sendMessage}},
+ {"SendMessageBatch", {EACLSourceType::QueueDir, sendMessage}},
+ {"ChangeMessageVisibility", {EACLSourceType::QueueDir, receiveMessage}},
+ {"ChangeMessageVisibilityBatch", {EACLSourceType::QueueDir, receiveMessage}},
+ {"ReceiveMessage", {EACLSourceType::QueueDir, receiveMessage}},
+ {"DeleteMessage", {EACLSourceType::QueueDir, deleteMessage}},
+ {"DeleteMessageBatch", {EACLSourceType::QueueDir, deleteMessage}},
+ {"PurgeQueue", {EACLSourceType::QueueDir, deleteMessage}},
+ {"ModifyPermissions", {EACLSourceType::Custom, modifyPermissions}},
+ {"SetQueueAttributes", {EACLSourceType::QueueDir, alterQueue}},
+ {"TagQueue", {EACLSourceType::QueueDir, alterQueue}},
+ {"UntagQueue", {EACLSourceType::QueueDir, alterQueue}},
+ {"GetQueueAttributes", {EACLSourceType::QueueDir, describePath}},
+ {"ListQueueTags", {EACLSourceType::QueueDir, describePath}},
+ {"GetQueueUrl", {EACLSourceType::QueueDir, describePath}},
+ {"ListQueues", {EACLSourceType::AccountDir, describePath}},
+ {"ListDeadLetterSourceQueues", {EACLSourceType::QueueDir, describePath}},
+ {"ListPermissions", {EACLSourceType::Custom, describePath}},
+ {"CountQueues", {EACLSourceType::AccountDir, describePath}},
+ {"DeleteUser", {EACLSourceType::AccountDir, deleteUser}},
+ {"DeleteQueue", {EACLSourceType::QueueDir, deleteQueue}}};
+
+ for (const auto& pair : ACE2AccessIndex) {
+ Access2ACEIndex[pair.second] = pair.first;
+ }
+
+ Y_VERIFY(Access2ACEIndex.size() == ACE2AccessIndex.size());
+ }
+
+ EACLSourceType GetActionACLSourceTypeImpl(const TStringBuf action) const {
+ auto it = Action2ACEIndex.find(action);
+ if (it != Action2ACEIndex.end()) {
+ return it->second.first;
+ }
+
+ return EACLSourceType::Unknown;
+ }
+
+ ui32 GetActionRequiredAccessImpl(const TStringBuf action) const {
+ return GetACERequiredAccessImpl(GetActionMatchingACEImpl(action));
+ }
+
+ ui32 GetACERequiredAccessImpl(const TStringBuf aceName) const {
+ auto accessIt = ACE2AccessIndex.find(aceName);
+ if (accessIt != ACE2AccessIndex.end()) {
+ return accessIt->second;
+ }
+
+ return 0;
+ }
+
+ TString GetActionMatchingACEImpl(const TStringBuf actionName) const {
+ auto it = Action2ACEIndex.find(actionName);
+ if (it != Action2ACEIndex.end()) {
+ return TString(it->second.second);
+ }
+
+ return {};
+ }
+
+ TVector<TStringBuf> GetAccessMatchingACEImpl(const ui32 access) const {
+ TVector<TStringBuf> result;
+ for (const auto& pair : Access2ACEIndex) {
+ if (pair.first & access) {
+ result.push_back(pair.second);
+ }
+ }
+
+ return result;
+ }
+
+private:
+ THashMap<TStringBuf, ui32> ACE2AccessIndex;
+ THashMap<ui32, TStringBuf> Access2ACEIndex;
+ THashMap<TStringBuf, std::pair<EACLSourceType, TStringBuf>> Action2ACEIndex;
+};
+
+static const TSQSACLMappings& Mappings() {
+ return *Singleton<TSQSACLMappings>();
+}
+
+EACLSourceType GetActionACLSourceType(const TString& actionName) {
+ return Mappings().GetActionACLSourceTypeImpl(actionName);
+}
+
+ui32 GetActionRequiredAccess(const TString& actionName) {
+ return Mappings().GetActionRequiredAccessImpl(actionName);
+}
+
+ui32 GetACERequiredAccess(const TString& aceName) {
+ return Mappings().GetACERequiredAccessImpl(aceName);
+}
+
+TString GetActionMatchingACE(const TString& actionName) {
+ return Mappings().GetActionMatchingACEImpl(actionName);
+}
+
+TVector<TStringBuf> GetAccessMatchingACE(const ui32 access) {
+ return Mappings().GetAccessMatchingACEImpl(access);
+}
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/acl.h b/ydb/core/ymq/base/acl.h
index 494881156c..18c3f07a39 100644
--- a/ydb/core/ymq/base/acl.h
+++ b/ydb/core/ymq/base/acl.h
@@ -1,21 +1,21 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/aclib/aclib.h>
-
+
namespace NKikimr::NSQS {
-enum class EACLSourceType : ui32 {
- Unknown,
- RootDir,
- AccountDir,
- QueueDir,
- Custom
-};
-
-EACLSourceType GetActionACLSourceType(const TString& actionName);
-ui32 GetActionRequiredAccess(const TString& actionName);
-ui32 GetACERequiredAccess(const TString& aceName);
-TString GetActionMatchingACE(const TString& actionName);
-TVector<TStringBuf> GetAccessMatchingACE(const ui32 access);
+enum class EACLSourceType : ui32 {
+ Unknown,
+ RootDir,
+ AccountDir,
+ QueueDir,
+ Custom
+};
+
+EACLSourceType GetActionACLSourceType(const TString& actionName);
+ui32 GetActionRequiredAccess(const TString& actionName);
+ui32 GetACERequiredAccess(const TString& aceName);
+TString GetActionMatchingACE(const TString& actionName);
+TVector<TStringBuf> GetAccessMatchingACE(const ui32 access);
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/action.h b/ydb/core/ymq/base/action.h
index decbb1775c..ede7922dd1 100644
--- a/ydb/core/ymq/base/action.h
+++ b/ydb/core/ymq/base/action.h
@@ -27,9 +27,9 @@ enum EAction {
SendMessageBatch,
SetQueueAttributes,
ModifyPermissions,
- ListPermissions,
- ListDeadLetterSourceQueues,
- CountQueues,
+ ListPermissions,
+ ListDeadLetterSourceQueues,
+ CountQueues,
ActionsArraySize,
};
@@ -63,7 +63,7 @@ EAction GetNonBatchAction(EAction action);
macro(ReceiveMessage) \
macro(SendMessage) \
macro(SendMessageBatch) \
- macro(ListDeadLetterSourceQueues) \
+ macro(ListDeadLetterSourceQueues) \
macro(SetQueueAttributes)
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/cloud_enums.h b/ydb/core/ymq/base/cloud_enums.h
index 2282ea049b..d0adc427e1 100644
--- a/ydb/core/ymq/base/cloud_enums.h
+++ b/ydb/core/ymq/base/cloud_enums.h
@@ -1,16 +1,16 @@
-#pragma once
-
-namespace NKikimr::NSQS::NCloudAuth {
- enum EActionType {
- Authenticate = 0,
- Authorize,
- GetCloudId,
- ActionTypesCount
- };
-
- enum ECredentialType {
- IamToken = 0,
- Signature,
- CredentialTypesCount
- };
-} // namespace NKikimr::NSQS::NCloudAuth
+#pragma once
+
+namespace NKikimr::NSQS::NCloudAuth {
+ enum EActionType {
+ Authenticate = 0,
+ Authorize,
+ GetCloudId,
+ ActionTypesCount
+ };
+
+ enum ECredentialType {
+ IamToken = 0,
+ Signature,
+ CredentialTypesCount
+ };
+} // namespace NKikimr::NSQS::NCloudAuth
diff --git a/ydb/core/ymq/base/constants.h b/ydb/core/ymq/base/constants.h
index c33488e62c..670e427ff8 100644
--- a/ydb/core/ymq/base/constants.h
+++ b/ydb/core/ymq/base/constants.h
@@ -1,8 +1,8 @@
#pragma once
#include <cstddef>
-#include <util/generic/string.h>
-
+#include <util/generic/string.h>
+
#define INFLY_LIMIT 120000
namespace NKikimr::NSQS {
@@ -12,5 +12,5 @@ constexpr size_t MAX_PARTITIONS_COUNT = 128;
static const TString yaSqsArnPrefix = "yrn:ya:sqs";
static const TString cloudArnPrefix = "yrn:yc:ymq";
-
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/counters.cpp b/ydb/core/ymq/base/counters.cpp
index 9b7db90966..90a9f5000f 100644
--- a/ydb/core/ymq/base/counters.cpp
+++ b/ydb/core/ymq/base/counters.cpp
@@ -298,8 +298,8 @@ static_assert(AbsDiffLessThanCounter(HttpUserCountersDescriptor.SizeOfCounters()
sizeof(NKikimrConfig::TSqsConfig*) +
sizeof(TIntrusivePtr<THttpUserCounters>), sizeof(THttpUserCounters)));
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup) {
- return GetServiceCounters(countersRoot, "sqs")->GetSubgroup("subsystem", subgroup);
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup) {
+ return GetServiceCounters(countersRoot, "sqs")->GetSubgroup("subsystem", subgroup);
}
TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot) {
// Remove subgroup and don't have subsystem (is this correct - ?)
@@ -605,7 +605,7 @@ void TQueueCounters::InitCounters(bool forLeaderNode) {
);
INIT_COUNTER(QueueCounters.SqsCounters, MessagesMovedToDLQ, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart);
-
+
INIT_COUNTERS_COUPLE(
QueueCounters,
SendMessage_DeduplicationCount, deduplicated_count_per_second,
@@ -881,63 +881,63 @@ void THttpActionCounters::SetAggregatedParent(THttpActionCounters* parent) {
HttpActionCountersDescriptor.SetAggregatedParent(this, parent);
}
-static const TString& StringifyGrpcStatus(int grpcStatus) {
- if (grpcStatus < 0 || grpcStatus > TCloudAuthCounters::GRPC_STATUSES_COUNT - 2) {
- grpcStatus = TCloudAuthCounters::GRPC_STATUSES_COUNT - 1;
- }
-
- static const TString statusStrings[] = {
- "Ok",
- "Cancelled",
- "Unknown",
- "InvalidArgument",
- "DeadlineExceeded",
- "NotFound",
- "AlreadyExists",
- "PermissionDenied",
- "ResourceExhausted",
- "FailedPrecondition",
- "Aborted",
- "OutOfRange",
- "Unimplemented",
- "Internal",
- "Unavailable",
- "DataLoss",
- "Unauthenticated",
- "Misc",
- };
-
- static_assert(Y_ARRAY_SIZE(statusStrings) == TCloudAuthCounters::GRPC_STATUSES_COUNT);
-
- return statusStrings[grpcStatus];
-}
-
-void TCloudAuthCounters::IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus) {
- if (grpcStatus < 0 || grpcStatus > GRPC_STATUSES_COUNT - 2) {
- grpcStatus = GRPC_STATUSES_COUNT - 1;
- }
-
- ++*CloudAuthCounters[actionType][credentialType][grpcStatus];
-}
-
-void TCloudAuthCounters::InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters) {
- for (size_t actionType = 0; actionType < NCloudAuth::EActionType::ActionTypesCount; ++actionType) {
- const auto actionTypeStr = ToString(static_cast<NCloudAuth::EActionType>(actionType));
- const auto actionCounters = cloudAuthCounters->GetSubgroup("action_type", actionTypeStr);
- for (size_t credentialType = 0; credentialType < NCloudAuth::ECredentialType::CredentialTypesCount; ++credentialType) {
- const auto credentialTypeStr = ToString(static_cast<NCloudAuth::ECredentialType>(credentialType));
- const auto actionAndCredentialCounters = actionCounters->GetSubgroup("credential_type", credentialTypeStr);
- for (size_t grpcStatus = 0; grpcStatus < GRPC_STATUSES_COUNT; ++grpcStatus) {
+static const TString& StringifyGrpcStatus(int grpcStatus) {
+ if (grpcStatus < 0 || grpcStatus > TCloudAuthCounters::GRPC_STATUSES_COUNT - 2) {
+ grpcStatus = TCloudAuthCounters::GRPC_STATUSES_COUNT - 1;
+ }
+
+ static const TString statusStrings[] = {
+ "Ok",
+ "Cancelled",
+ "Unknown",
+ "InvalidArgument",
+ "DeadlineExceeded",
+ "NotFound",
+ "AlreadyExists",
+ "PermissionDenied",
+ "ResourceExhausted",
+ "FailedPrecondition",
+ "Aborted",
+ "OutOfRange",
+ "Unimplemented",
+ "Internal",
+ "Unavailable",
+ "DataLoss",
+ "Unauthenticated",
+ "Misc",
+ };
+
+ static_assert(Y_ARRAY_SIZE(statusStrings) == TCloudAuthCounters::GRPC_STATUSES_COUNT);
+
+ return statusStrings[grpcStatus];
+}
+
+void TCloudAuthCounters::IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus) {
+ if (grpcStatus < 0 || grpcStatus > GRPC_STATUSES_COUNT - 2) {
+ grpcStatus = GRPC_STATUSES_COUNT - 1;
+ }
+
+ ++*CloudAuthCounters[actionType][credentialType][grpcStatus];
+}
+
+void TCloudAuthCounters::InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters) {
+ for (size_t actionType = 0; actionType < NCloudAuth::EActionType::ActionTypesCount; ++actionType) {
+ const auto actionTypeStr = ToString(static_cast<NCloudAuth::EActionType>(actionType));
+ const auto actionCounters = cloudAuthCounters->GetSubgroup("action_type", actionTypeStr);
+ for (size_t credentialType = 0; credentialType < NCloudAuth::ECredentialType::CredentialTypesCount; ++credentialType) {
+ const auto credentialTypeStr = ToString(static_cast<NCloudAuth::ECredentialType>(credentialType));
+ const auto actionAndCredentialCounters = actionCounters->GetSubgroup("credential_type", credentialTypeStr);
+ for (size_t grpcStatus = 0; grpcStatus < GRPC_STATUSES_COUNT; ++grpcStatus) {
INIT_COUNTER_WITH_NAME(actionAndCredentialCounters, CloudAuthCounters[actionType][credentialType][grpcStatus], StringifyGrpcStatus(grpcStatus), ELifetime::Persistent, EValueType::Derivative, Lazy(*Cfg));
- }
- }
- }
+ }
+ }
+ }
INIT_HISTOGRAM_COUNTER(cloudAuthCounters, AuthenticateDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg));
INIT_HISTOGRAM_COUNTER(cloudAuthCounters, AuthorizeDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg));
INIT_HISTOGRAM_COUNTER(cloudAuthCounters, GetFolderIdDuration, ELifetime::Persistent, DurationBucketsMs, Lazy(*Cfg));
-}
-
+}
+
void TMeteringCounters::InitCounters(const TVector<TString>& classifierLabels) {
const TString classifierRequestsLabel("ClassifierRequests_");
const TString idleClassifierRequestsLabel("IdleClassifierRequests_");
diff --git a/ydb/core/ymq/base/counters.h b/ydb/core/ymq/base/counters.h
index 841afdc99f..3c2667b241 100644
--- a/ydb/core/ymq/base/counters.h
+++ b/ydb/core/ymq/base/counters.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include <ydb/core/protos/config.pb.h>
#include <ydb/core/ymq/base/action.h>
@@ -22,7 +22,7 @@ namespace NKikimr::NSQS {
struct TUserCounters;
struct TQueueCounters;
struct THttpCounters;
-struct TCloudAuthCounters;
+struct TCloudAuthCounters;
#define INC_COUNTER(countersPack, counter) \
if (countersPack) { \
@@ -587,7 +587,7 @@ struct TQueueCounters : public TAtomicRefCount<TQueueCounters> {
TLazyCachedCounter received_bytes_per_second;
TLazyCachedCounter MessagesMovedToDLQ; // Count of messages that were moved to DLQ.
-
+
TLazyCachedCounter SendMessage_DeduplicationCount; // Count of messages that were deduplicated (for fifo only).
TLazyCachedCounter deduplicated_count_per_second;
TLazyCachedCounter SendMessage_Count; // Count of messages that were sent. // User metric for cloud console.
@@ -762,7 +762,7 @@ private:
};
// Cloud specific counters.
-struct TCloudAuthCounters {
+struct TCloudAuthCounters {
// Durations for different security actions types.
TLazyCachedHistogram AuthenticateDuration; // Histogram with buckets for durations (== 18 counters).
TLazyCachedHistogram AuthorizeDuration; // Histogram with buckets for durations (== 18 counters).
@@ -771,23 +771,23 @@ struct TCloudAuthCounters {
explicit TCloudAuthCounters(const NKikimrConfig::TSqsConfig& cfg, TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCountersRoot)
: Cfg(&cfg)
{
- InitCounters(std::move(cloudAuthCountersRoot));
- }
-
- void IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus);
-
- static constexpr int GRPC_STATUSES_COUNT = 18;
-
-private:
- void InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters);
-
-private:
+ InitCounters(std::move(cloudAuthCountersRoot));
+ }
+
+ void IncCounter(const NCloudAuth::EActionType actionType, const NCloudAuth::ECredentialType credentialType, int grpcStatus);
+
+ static constexpr int GRPC_STATUSES_COUNT = 18;
+
+private:
+ void InitCounters(TIntrusivePtr<NMonitoring::TDynamicCounters> cloudAuthCounters);
+
+private:
const NKikimrConfig::TSqsConfig* Cfg = nullptr;
TLazyCachedCounter CloudAuthCounters[NCloudAuth::EActionType::ActionTypesCount] // 3 types.
[NCloudAuth::ECredentialType::CredentialTypesCount] // 2 types.
[GRPC_STATUSES_COUNT]; // 18 types.
-};
-
+};
+
// Metering counters in SQS core subsystem.
struct TMeteringCounters : public TAtomicRefCount<TMeteringCounters> {
THashMap<TString, TLazyCachedCounter> ClassifierRequestsResults;
diff --git a/ydb/core/ymq/base/dlq_helpers.cpp b/ydb/core/ymq/base/dlq_helpers.cpp
index a59dcb417c..a2ba9fac42 100644
--- a/ydb/core/ymq/base/dlq_helpers.cpp
+++ b/ydb/core/ymq/base/dlq_helpers.cpp
@@ -1,105 +1,105 @@
-#include "dlq_helpers.h"
-
+#include "dlq_helpers.h"
+
#include <ydb/core/ymq/base/constants.h>
#include <ydb/core/ymq/base/limits.h>
#include <library/cpp/scheme/scheme.h>
-
+
#include <util/string/split.h>
-#include <util/string/join.h>
-
-namespace NKikimr::NSQS {
-
-static const TStringBuf maxReceiveCount = "maxReceiveCount";
-static const TStringBuf deadLetterTargetArn = "deadLetterTargetArn";
-
-static bool IsValidArn(const TString& arn, const NKikimrConfig::TSqsConfig& config) {
- TVector<TStringBuf> items;
- StringSplitter(arn).Split(':').AddTo(&items);
- if (items.size() != 6) {
- return false;
- }
-
- // prefix
- const TString& requiredPrefix = config.GetYandexCloudMode() ? cloudArnPrefix : yaSqsArnPrefix;
- if (!arn.StartsWith(requiredPrefix)) {
- return false;
- }
-
- // region
- if (items[3] != config.GetYandexCloudServiceRegion()) {
- return false;
- }
-
- return items[4] && items[5]; // account | folder id and queue name
-}
-
-TRedrivePolicy TRedrivePolicy::FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config) {
-
- TRedrivePolicy policy;
- const NSc::TValue validatedRedrivePolicy = NSc::TValue::FromJson(json);
- if (validatedRedrivePolicy.IsDict() &&
- validatedRedrivePolicy.Has(maxReceiveCount) && (validatedRedrivePolicy[maxReceiveCount].IsString() || validatedRedrivePolicy[maxReceiveCount].IsIntNumber()) &&
- validatedRedrivePolicy.Has(deadLetterTargetArn) && validatedRedrivePolicy[deadLetterTargetArn].IsString())
- {
- size_t maxReceiveCountValue = 0;
- if (TryFromString(validatedRedrivePolicy[maxReceiveCount].ForceString(), maxReceiveCountValue)) {
- if (maxReceiveCountValue < TLimits::MinMaxReceiveCount || maxReceiveCountValue > TLimits::MaxMaxReceiveCount) {
+#include <util/string/join.h>
+
+namespace NKikimr::NSQS {
+
+static const TStringBuf maxReceiveCount = "maxReceiveCount";
+static const TStringBuf deadLetterTargetArn = "deadLetterTargetArn";
+
+static bool IsValidArn(const TString& arn, const NKikimrConfig::TSqsConfig& config) {
+ TVector<TStringBuf> items;
+ StringSplitter(arn).Split(':').AddTo(&items);
+ if (items.size() != 6) {
+ return false;
+ }
+
+ // prefix
+ const TString& requiredPrefix = config.GetYandexCloudMode() ? cloudArnPrefix : yaSqsArnPrefix;
+ if (!arn.StartsWith(requiredPrefix)) {
+ return false;
+ }
+
+ // region
+ if (items[3] != config.GetYandexCloudServiceRegion()) {
+ return false;
+ }
+
+ return items[4] && items[5]; // account | folder id and queue name
+}
+
+TRedrivePolicy TRedrivePolicy::FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config) {
+
+ TRedrivePolicy policy;
+ const NSc::TValue validatedRedrivePolicy = NSc::TValue::FromJson(json);
+ if (validatedRedrivePolicy.IsDict() &&
+ validatedRedrivePolicy.Has(maxReceiveCount) && (validatedRedrivePolicy[maxReceiveCount].IsString() || validatedRedrivePolicy[maxReceiveCount].IsIntNumber()) &&
+ validatedRedrivePolicy.Has(deadLetterTargetArn) && validatedRedrivePolicy[deadLetterTargetArn].IsString())
+ {
+ size_t maxReceiveCountValue = 0;
+ if (TryFromString(validatedRedrivePolicy[maxReceiveCount].ForceString(), maxReceiveCountValue)) {
+ if (maxReceiveCountValue < TLimits::MinMaxReceiveCount || maxReceiveCountValue > TLimits::MaxMaxReceiveCount) {
policy.ErrorText = "maxReceiveCount must be greater than 0 and less than 1001.";
- return policy;
- }
- } else {
+ return policy;
+ }
+ } else {
policy.ErrorText = "Failed to parse maxReceiveCount as an integer.";
- return policy;
- }
-
- const TString arn = validatedRedrivePolicy[deadLetterTargetArn].ForceString();
- if (!IsValidArn(arn, config)) {
- policy.ErrorText = "Invalid deadLetterTargetArn.";
- return policy;
- }
-
- TStringBuf skipped, queueName;
- if (!TStringBuf(arn).TryRSplit(':', skipped, queueName)) {
- queueName = arn;
- }
-
- if (!queueName) {
+ return policy;
+ }
+
+ const TString arn = validatedRedrivePolicy[deadLetterTargetArn].ForceString();
+ if (!IsValidArn(arn, config)) {
+ policy.ErrorText = "Invalid deadLetterTargetArn.";
+ return policy;
+ }
+
+ TStringBuf skipped, queueName;
+ if (!TStringBuf(arn).TryRSplit(':', skipped, queueName)) {
+ queueName = arn;
+ }
+
+ if (!queueName) {
policy.ErrorText = "Empty dead letter target queue name.";
- return policy;
- }
-
- policy.MaxReceiveCount = maxReceiveCountValue;
- policy.TargetQueueName = TString(queueName);
- policy.TargetArn = TString(arn);
- } else if (!json) {
- // Clear redrive policy
- policy.MaxReceiveCount = 0;
- policy.TargetQueueName = "";
- policy.TargetArn = "";
- } else {
+ return policy;
+ }
+
+ policy.MaxReceiveCount = maxReceiveCountValue;
+ policy.TargetQueueName = TString(queueName);
+ policy.TargetArn = TString(arn);
+ } else if (!json) {
+ // Clear redrive policy
+ policy.MaxReceiveCount = 0;
+ policy.TargetQueueName = "";
+ policy.TargetArn = "";
+ } else {
policy.ErrorText = "RedrivePolicy attribute is ill-constructed.";
- }
-
- return policy;
-}
-
-TString TRedrivePolicy::ToJson() const {
- if (MaxReceiveCount && TargetArn) {
- NSc::TValue policy;
- policy.SetDict();
- policy[maxReceiveCount] = *MaxReceiveCount;
- policy[deadLetterTargetArn] = *TargetArn;
- return policy.ToJson();
- }
- return {};
-}
-
-bool TRedrivePolicy::IsValid() const {
- return ErrorText.empty();
-}
-
-const TString& TRedrivePolicy::GetErrorText() const {
- return ErrorText;
-}
-
-} // namespace NKikimr::NSQS
+ }
+
+ return policy;
+}
+
+TString TRedrivePolicy::ToJson() const {
+ if (MaxReceiveCount && TargetArn) {
+ NSc::TValue policy;
+ policy.SetDict();
+ policy[maxReceiveCount] = *MaxReceiveCount;
+ policy[deadLetterTargetArn] = *TargetArn;
+ return policy.ToJson();
+ }
+ return {};
+}
+
+bool TRedrivePolicy::IsValid() const {
+ return ErrorText.empty();
+}
+
+const TString& TRedrivePolicy::GetErrorText() const {
+ return ErrorText;
+}
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/dlq_helpers.h b/ydb/core/ymq/base/dlq_helpers.h
index 968eb87f73..9c2e1d292b 100644
--- a/ydb/core/ymq/base/dlq_helpers.h
+++ b/ydb/core/ymq/base/dlq_helpers.h
@@ -1,23 +1,23 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/config.pb.h>
-
-#include <util/generic/maybe.h>
-#include <util/generic/string.h>
-
-namespace NKikimr::NSQS {
-
-struct TRedrivePolicy {
- static TRedrivePolicy FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config);
-
- TString ToJson() const;
- bool IsValid() const;
- const TString& GetErrorText() const;
-
- TString ErrorText;
- TMaybe<size_t> MaxReceiveCount;
- TMaybe<TString> TargetArn;
- TMaybe<TString> TargetQueueName;
-};
-
-} // namespace NKikimr::NSQS
+
+#include <util/generic/maybe.h>
+#include <util/generic/string.h>
+
+namespace NKikimr::NSQS {
+
+struct TRedrivePolicy {
+ static TRedrivePolicy FromJson(const TString& json, const NKikimrConfig::TSqsConfig& config);
+
+ TString ToJson() const;
+ bool IsValid() const;
+ const TString& GetErrorText() const;
+
+ TString ErrorText;
+ TMaybe<size_t> MaxReceiveCount;
+ TMaybe<TString> TargetArn;
+ TMaybe<TString> TargetQueueName;
+};
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/limits.h b/ydb/core/ymq/base/limits.h
index 6762fa9cf1..b251b3ff62 100644
--- a/ydb/core/ymq/base/limits.h
+++ b/ydb/core/ymq/base/limits.h
@@ -1,6 +1,6 @@
#pragma once
-#include <util/datetime/base.h>
+#include <util/datetime/base.h>
#include <util/system/defaults.h>
namespace NKikimr::NSQS {
@@ -25,10 +25,10 @@ namespace TLimits {
static constexpr size_t VisibilityTimeout = 30;
static constexpr TDuration MaxVisibilityTimeout = TDuration::Hours(12);
-
- static constexpr size_t MinMaxReceiveCount = 1;
-
- static constexpr size_t MaxMaxReceiveCount = 1000;
+
+ static constexpr size_t MinMaxReceiveCount = 1;
+
+ static constexpr size_t MaxMaxReceiveCount = 1000;
};
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/processed_request_attributes.h b/ydb/core/ymq/base/processed_request_attributes.h
index 2892b899ee..9a5169b3ab 100644
--- a/ydb/core/ymq/base/processed_request_attributes.h
+++ b/ydb/core/ymq/base/processed_request_attributes.h
@@ -1,26 +1,26 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/ymq/base/action.h>
-
-#include <util/generic/string.h>
-
-namespace NKikimr::NSQS {
-
-struct TProcessedRequestAttributes {
- TProcessedRequestAttributes() = default;
- TProcessedRequestAttributes(TProcessedRequestAttributes&& rhs) = default;
-
- bool IsFifo = false;
- int HttpStatusCode = 0;
-
- ui64 RequestSizeInBytes = 0;
- ui64 ResponseSizeInBytes = 0;
-
- TString FolderId;
- TString ResourceId;
- TString SourceAddress;
-
- EAction Action;
-};
-
-} // namespace NKikimr::NSQS
+
+#include <util/generic/string.h>
+
+namespace NKikimr::NSQS {
+
+struct TProcessedRequestAttributes {
+ TProcessedRequestAttributes() = default;
+ TProcessedRequestAttributes(TProcessedRequestAttributes&& rhs) = default;
+
+ bool IsFifo = false;
+ int HttpStatusCode = 0;
+
+ ui64 RequestSizeInBytes = 0;
+ ui64 ResponseSizeInBytes = 0;
+
+ TString FolderId;
+ TString ResourceId;
+ TString SourceAddress;
+
+ EAction Action;
+};
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/query_id.h b/ydb/core/ymq/base/query_id.h
index d313c432f8..ad54d0c70d 100644
--- a/ydb/core/ymq/base/query_id.h
+++ b/ydb/core/ymq/base/query_id.h
@@ -25,9 +25,9 @@ enum EQueryId {
LOAD_INFLY_ID = 17,
ADD_MESSAGES_TO_INFLY_ID = 18,
GET_STATE_ID = 19,
- LIST_DEAD_LETTER_SOURCE_QUEUES_ID = 20,
- LOAD_OR_REDRIVE_MESSAGE_ID = 21,
- READ_OR_REDRIVE_MESSAGE_ID = 22,
+ LIST_DEAD_LETTER_SOURCE_QUEUES_ID = 20,
+ LOAD_OR_REDRIVE_MESSAGE_ID = 21,
+ READ_OR_REDRIVE_MESSAGE_ID = 22,
GET_USER_SETTINGS_ID = 23,
GET_QUEUES_LIST_ID = 24,
diff --git a/ydb/core/ymq/base/queue_attributes.cpp b/ydb/core/ymq/base/queue_attributes.cpp
index 4f71ba1d47..8c6436aba9 100644
--- a/ydb/core/ymq/base/queue_attributes.cpp
+++ b/ydb/core/ymq/base/queue_attributes.cpp
@@ -1,115 +1,115 @@
-#include "limits.h"
-#include "queue_attributes.h"
-
-#include <util/generic/utility.h>
-#include <util/string/cast.h>
-#include <util/string/printf.h>
-
-namespace NKikimr::NSQS {
-
-TQueueAttributes TQueueAttributes::FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp) {
- TQueueAttributes result;
-
-#define INVALID_PARAM_MSG "Invalid value for the parameter %s."
-
-#define TRY_SET_ATTRIBUTE(NAME, MIN_VAL, MAX_VAL) \
- if (name == Y_STRINGIZE(NAME)) { \
- if (!result.TryParseLimitedValue(Y_STRINGIZE(NAME), \
- value, \
- MIN_VAL, \
- MAX_VAL, \
- result.NAME, \
- clamp)) \
- { \
- break; \
- } \
- }
-
- for (const auto& [name, value] : attributes) {
- TRY_SET_ATTRIBUTE(VisibilityTimeout,
- 0,
- TLimits::MaxVisibilityTimeout.Seconds())
- else TRY_SET_ATTRIBUTE(MessageRetentionPeriod,
- TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(),
- TLimits::MaxMessageRetentionPeriod.Seconds())
- else TRY_SET_ATTRIBUTE(ReceiveMessageWaitTimeSeconds,
- 0,
- TDuration::MilliSeconds(config.GetMaxWaitTimeoutMs()).Seconds())
- else TRY_SET_ATTRIBUTE(DelaySeconds,
- 0,
- TLimits::MaxDelaySeconds)
- else TRY_SET_ATTRIBUTE(MaximumMessageSize,
- 1024,
- TLimits::MaxMessageSize)
- else if (name == "ContentBasedDeduplication" && isFifoQueue) {
- if (value == "true") {
- result.ContentBasedDeduplication = true;
- } else if (value == "false") {
- result.ContentBasedDeduplication = false;
- } else {
- result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE;
+#include "limits.h"
+#include "queue_attributes.h"
+
+#include <util/generic/utility.h>
+#include <util/string/cast.h>
+#include <util/string/printf.h>
+
+namespace NKikimr::NSQS {
+
+TQueueAttributes TQueueAttributes::FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp) {
+ TQueueAttributes result;
+
+#define INVALID_PARAM_MSG "Invalid value for the parameter %s."
+
+#define TRY_SET_ATTRIBUTE(NAME, MIN_VAL, MAX_VAL) \
+ if (name == Y_STRINGIZE(NAME)) { \
+ if (!result.TryParseLimitedValue(Y_STRINGIZE(NAME), \
+ value, \
+ MIN_VAL, \
+ MAX_VAL, \
+ result.NAME, \
+ clamp)) \
+ { \
+ break; \
+ } \
+ }
+
+ for (const auto& [name, value] : attributes) {
+ TRY_SET_ATTRIBUTE(VisibilityTimeout,
+ 0,
+ TLimits::MaxVisibilityTimeout.Seconds())
+ else TRY_SET_ATTRIBUTE(MessageRetentionPeriod,
+ TDuration::MilliSeconds(config.GetMinMessageRetentionPeriodMs()).Seconds(),
+ TLimits::MaxMessageRetentionPeriod.Seconds())
+ else TRY_SET_ATTRIBUTE(ReceiveMessageWaitTimeSeconds,
+ 0,
+ TDuration::MilliSeconds(config.GetMaxWaitTimeoutMs()).Seconds())
+ else TRY_SET_ATTRIBUTE(DelaySeconds,
+ 0,
+ TLimits::MaxDelaySeconds)
+ else TRY_SET_ATTRIBUTE(MaximumMessageSize,
+ 1024,
+ TLimits::MaxMessageSize)
+ else if (name == "ContentBasedDeduplication" && isFifoQueue) {
+ if (value == "true") {
+ result.ContentBasedDeduplication = true;
+ } else if (value == "false") {
+ result.ContentBasedDeduplication = false;
+ } else {
+ result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE;
result.ErrorText = Sprintf(INVALID_PARAM_MSG " Valid values: true, false.", name.c_str());
- break;
- }
- } else if (name == "FifoQueue" && isFifoQueue) {
- if (value == "true") {
- continue; // just ignore the value
- } else {
- result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE;
- if (value == "false") {
+ break;
+ }
+ } else if (name == "FifoQueue" && isFifoQueue) {
+ if (value == "true") {
+ continue; // just ignore the value
+ } else {
+ result.Error = &NErrors::INVALID_ATTRIBUTE_VALUE;
+ if (value == "false") {
result.ErrorText = Sprintf(INVALID_PARAM_MSG " Reason: Modifying queue type is not supported.", name.c_str());
- } else {
+ } else {
result.ErrorText = Sprintf(INVALID_PARAM_MSG " Valid values: true, false.", name.c_str());
- }
- break;
- }
- } else if (config.GetEnableDeadLetterQueues() && name == "RedrivePolicy") {
- result.RedrivePolicy = TRedrivePolicy::FromJson(value, config);
- if (result.RedrivePolicy.IsValid()) {
- if (*result.RedrivePolicy.TargetQueueName && isFifoQueue != result.RedrivePolicy.TargetQueueName->EndsWith(".fifo")) {
+ }
+ break;
+ }
+ } else if (config.GetEnableDeadLetterQueues() && name == "RedrivePolicy") {
+ result.RedrivePolicy = TRedrivePolicy::FromJson(value, config);
+ if (result.RedrivePolicy.IsValid()) {
+ if (*result.RedrivePolicy.TargetQueueName && isFifoQueue != result.RedrivePolicy.TargetQueueName->EndsWith(".fifo")) {
result.ErrorText = "Target dead letter queue should have the same type as source queue.";
- } else {
- continue;
- }
- } else {
- result.ErrorText = result.RedrivePolicy.GetErrorText();
- }
- result.Error = &NErrors::INVALID_PARAMETER_VALUE;
- break;
- } else {
- result.Error = &NErrors::INVALID_ATTRIBUTE_NAME;
+ } else {
+ continue;
+ }
+ } else {
+ result.ErrorText = result.RedrivePolicy.GetErrorText();
+ }
+ result.Error = &NErrors::INVALID_PARAMETER_VALUE;
+ break;
+ } else {
+ result.Error = &NErrors::INVALID_ATTRIBUTE_NAME;
result.ErrorText = Sprintf("Unknown Attribute %s.", name.c_str());
- break;
- }
- }
-
- return result;
-}
-
-bool TQueueAttributes::Validate() const {
- return !Error;
-}
-
-bool TQueueAttributes::HasClampedAttributes() const {
- return Clamped;
-}
-
-bool TQueueAttributes::TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp) {
- ui64 value;
- if (TryFromString<ui64>(valueStr, value)) {
- const bool valid = (value >= allowedMinValue && value <= allowedMaxValue);
- if (valid || clamp) {
- result = valid ? value : ClampVal(value, allowedMinValue, allowedMaxValue);
- Clamped |= !valid;
- return true;
- }
- }
-
- result = Nothing();
- Error = &NErrors::INVALID_ATTRIBUTE_VALUE;
+ break;
+ }
+ }
+
+ return result;
+}
+
+bool TQueueAttributes::Validate() const {
+ return !Error;
+}
+
+bool TQueueAttributes::HasClampedAttributes() const {
+ return Clamped;
+}
+
+bool TQueueAttributes::TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp) {
+ ui64 value;
+ if (TryFromString<ui64>(valueStr, value)) {
+ const bool valid = (value >= allowedMinValue && value <= allowedMaxValue);
+ if (valid || clamp) {
+ result = valid ? value : ClampVal(value, allowedMinValue, allowedMaxValue);
+ Clamped |= !valid;
+ return true;
+ }
+ }
+
+ result = Nothing();
+ Error = &NErrors::INVALID_ATTRIBUTE_VALUE;
ErrorText = Sprintf(INVALID_PARAM_MSG " Valid values are from %" PRIu64 " to %" PRIu64 " both inclusive.", attrName.c_str(), allowedMinValue, allowedMaxValue);
-
- return false;
-}
-
-} // namespace NKikimr::NQS
+
+ return false;
+}
+
+} // namespace NKikimr::NQS
diff --git a/ydb/core/ymq/base/queue_attributes.h b/ydb/core/ymq/base/queue_attributes.h
index 645b9bff9c..3e0dfd19e0 100644
--- a/ydb/core/ymq/base/queue_attributes.h
+++ b/ydb/core/ymq/base/queue_attributes.h
@@ -1,34 +1,34 @@
-#pragma once
-
-#include "dlq_helpers.h"
+#pragma once
+
+#include "dlq_helpers.h"
#include <ydb/library/http_proxy/error/error.h>
-
+
#include <ydb/core/protos/config.pb.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/maybe.h>
-#include <util/generic/string.h>
-
-namespace NKikimr::NSQS {
-
-struct TQueueAttributes {
- static TQueueAttributes FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp);
- bool Validate() const;
- bool HasClampedAttributes() const;
-public:
- bool Clamped = false;
- const TErrorClass* Error = nullptr;
- TString ErrorText;
- TMaybe<ui64> VisibilityTimeout; // seconds
- TMaybe<ui64> MessageRetentionPeriod; // seconds
- TMaybe<ui64> ReceiveMessageWaitTimeSeconds; // seconds
- TMaybe<ui64> DelaySeconds; // seconds
- TMaybe<ui64> MaximumMessageSize; // bytes
- TMaybe<bool> ContentBasedDeduplication; // bool
- TRedrivePolicy RedrivePolicy;
-
-private:
- bool TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp);
-};
-
-} // namespace NKikimr::NSQS
+
+#include <util/generic/hash.h>
+#include <util/generic/maybe.h>
+#include <util/generic/string.h>
+
+namespace NKikimr::NSQS {
+
+struct TQueueAttributes {
+ static TQueueAttributes FromAttributesAndConfig(const THashMap<TString, TString>& attributes, const NKikimrConfig::TSqsConfig& config, bool isFifoQueue, bool clamp);
+ bool Validate() const;
+ bool HasClampedAttributes() const;
+public:
+ bool Clamped = false;
+ const TErrorClass* Error = nullptr;
+ TString ErrorText;
+ TMaybe<ui64> VisibilityTimeout; // seconds
+ TMaybe<ui64> MessageRetentionPeriod; // seconds
+ TMaybe<ui64> ReceiveMessageWaitTimeSeconds; // seconds
+ TMaybe<ui64> DelaySeconds; // seconds
+ TMaybe<ui64> MaximumMessageSize; // bytes
+ TMaybe<bool> ContentBasedDeduplication; // bool
+ TRedrivePolicy RedrivePolicy;
+
+private:
+ bool TryParseLimitedValue(const TString& attrName, const TString& valueStr, const ui64 allowedMinValue, const ui64 allowedMaxValue, TMaybe<ui64>& result, bool clamp);
+};
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/queue_id.cpp b/ydb/core/ymq/base/queue_id.cpp
index 2f0324efe2..0b93440a39 100644
--- a/ydb/core/ymq/base/queue_id.cpp
+++ b/ydb/core/ymq/base/queue_id.cpp
@@ -1,74 +1,74 @@
-#include <util/generic/hash.h>
-#include <util/generic/string.h>
-#include <util/string/cast.h>
-
+#include <util/generic/hash.h>
+#include <util/generic/string.h>
+#include <util/string/cast.h>
+
namespace NKikimr::NSQS {
-
-// Queue id breakdown in bits [15][1][64][7][13]:
-// [15] - service id
-// [1] - reserved == 0
-// [64] - globally unique number
-// [7] - reserved == 0
-// [13] - partial hash result for account name (it's not really important to have stable hash impl)
-// total: 15 + 1 + 64 + 7 + 13 == 100
-
-class TQueueId {
-public:
- TQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) {
- char* bufPtr = Buf;
-
- // first 15 bits
- Y_VERIFY(serviceId <= 0x7FFF);
- ConvertAndAppendLastNBits(serviceId, 15, bufPtr);
-
- // next 65 bits (1 bit is reserved and 64 are the place for the unique num)
- *bufPtr++ = SmallIntToBase32(uniqueNum >> 60); // first 5 bits
- ConvertAndAppendLastNBits(uniqueNum, 60, bufPtr); // 60 other bits
-
- // last 20 bits
- const ui32 appendix = THash<TString>()(accountName) & 0x1FFF; // use last 13 bits only
- ConvertAndAppendLastNBits(appendix, 20, bufPtr);
-
- Y_VERIFY(bufPtr == &Buf[20]);
-
- *bufPtr = '\0';
- }
-
- TString AsString() const {
- return TString(Buf);
- }
-
-private:
- static const ui64 BasicMask = 0x1F; // 11111 as binary
- static const ui8 Radix = 5;
-
- char Buf[21] = {};
-
-private:
- char SmallIntToBase32(const ui8 v) const {
- Y_VERIFY(v < 32);
-
- if (v < 10) {
- return '0' + v;
- } else {
- return 'a' + v - 10;
- }
- }
-
- void ConvertAndAppendLastNBits(const ui64 num, const ui8 n, char*& output) const {
- Y_VERIFY(n % Radix == 0);
-
- for (int i = n / Radix - 1; i >= 0; --i) {
- const ui8 shift = i * Radix;
- const ui64 mask = BasicMask << shift;
-
- *output++ = SmallIntToBase32((num & mask) >> shift);
- }
- }
-};
-
-TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) {
- return TQueueId(serviceId, uniqueNum, accountName).AsString();
-}
-
+
+// Queue id breakdown in bits [15][1][64][7][13]:
+// [15] - service id
+// [1] - reserved == 0
+// [64] - globally unique number
+// [7] - reserved == 0
+// [13] - partial hash result for account name (it's not really important to have stable hash impl)
+// total: 15 + 1 + 64 + 7 + 13 == 100
+
+class TQueueId {
+public:
+ TQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) {
+ char* bufPtr = Buf;
+
+ // first 15 bits
+ Y_VERIFY(serviceId <= 0x7FFF);
+ ConvertAndAppendLastNBits(serviceId, 15, bufPtr);
+
+ // next 65 bits (1 bit is reserved and 64 are the place for the unique num)
+ *bufPtr++ = SmallIntToBase32(uniqueNum >> 60); // first 5 bits
+ ConvertAndAppendLastNBits(uniqueNum, 60, bufPtr); // 60 other bits
+
+ // last 20 bits
+ const ui32 appendix = THash<TString>()(accountName) & 0x1FFF; // use last 13 bits only
+ ConvertAndAppendLastNBits(appendix, 20, bufPtr);
+
+ Y_VERIFY(bufPtr == &Buf[20]);
+
+ *bufPtr = '\0';
+ }
+
+ TString AsString() const {
+ return TString(Buf);
+ }
+
+private:
+ static const ui64 BasicMask = 0x1F; // 11111 as binary
+ static const ui8 Radix = 5;
+
+ char Buf[21] = {};
+
+private:
+ char SmallIntToBase32(const ui8 v) const {
+ Y_VERIFY(v < 32);
+
+ if (v < 10) {
+ return '0' + v;
+ } else {
+ return 'a' + v - 10;
+ }
+ }
+
+ void ConvertAndAppendLastNBits(const ui64 num, const ui8 n, char*& output) const {
+ Y_VERIFY(n % Radix == 0);
+
+ for (int i = n / Radix - 1; i >= 0; --i) {
+ const ui8 shift = i * Radix;
+ const ui64 mask = BasicMask << shift;
+
+ *output++ = SmallIntToBase32((num & mask) >> shift);
+ }
+ }
+};
+
+TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName) {
+ return TQueueId(serviceId, uniqueNum, accountName).AsString();
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/queue_id.h b/ydb/core/ymq/base/queue_id.h
index 11ce258b4c..a2cac100e0 100644
--- a/ydb/core/ymq/base/queue_id.h
+++ b/ydb/core/ymq/base/queue_id.h
@@ -1,9 +1,9 @@
-#pragma once
-
-#include <util/generic/string.h>
-
+#pragma once
+
+#include <util/generic/string.h>
+
namespace NKikimr::NSQS {
-
-TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName);
-
+
+TString MakeQueueId(const ui16 serviceId, const ui64 uniqueNum, const TString& accountName);
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/queue_path.h b/ydb/core/ymq/base/queue_path.h
index dcf08c3c8b..baaa781b00 100644
--- a/ydb/core/ymq/base/queue_path.h
+++ b/ydb/core/ymq/base/queue_path.h
@@ -8,7 +8,7 @@ struct TQueuePath {
TString Root;
TString UserName;
TString QueueName;
- TString VersionSuffix;
+ TString VersionSuffix;
TQueuePath()
{ }
@@ -20,11 +20,11 @@ struct TQueuePath {
: Root(root)
, UserName(userName)
, QueueName(queueName)
- {
- if (version) {
- VersionSuffix = TString::Join("v", ToString(version));
- }
- }
+ {
+ if (version) {
+ VersionSuffix = TString::Join("v", ToString(version));
+ }
+ }
operator TString() const {
return GetQueuePath();
@@ -35,14 +35,14 @@ struct TQueuePath {
}
TString GetVersionedQueuePath() const {
- auto tmp = GetQueuePath();
- if (!VersionSuffix) {
- return tmp;
- } else {
- return Join("/", tmp, VersionSuffix);
- }
- }
-
+ auto tmp = GetQueuePath();
+ if (!VersionSuffix) {
+ return tmp;
+ } else {
+ return Join("/", tmp, VersionSuffix);
+ }
+ }
+
TString GetUserPath() const {
return Join("/", Root, UserName);
}
diff --git a/ydb/core/ymq/base/security.h b/ydb/core/ymq/base/security.h
index 58a1f6afc3..764be96265 100644
--- a/ydb/core/ymq/base/security.h
+++ b/ydb/core/ymq/base/security.h
@@ -1,30 +1,30 @@
-#pragma once
-
-#include <util/generic/string.h>
-
-
+#pragma once
+
+#include <util/generic/string.h>
+
+
namespace NKikimr::NSQS {
-
-template <typename TReq, typename TCreds>
-TString ExtractSecurityToken(const TReq& request) {
- if (request.HasCredentials()) {
- switch (request.GetCredentials().AccessKey_case()) {
- case TCreds::kOAuthToken: {
- return request.GetCredentials().GetOAuthToken();
- }
- case TCreds::kTvmTicket: {
- return request.GetCredentials().GetTvmTicket();
- }
- case TCreds::kStaticCreds: {
- return request.GetCredentials().GetStaticCreds();
- }
- default: {
- // leave the token empty
- }
- }
- }
-
- return {};
-}
-
+
+template <typename TReq, typename TCreds>
+TString ExtractSecurityToken(const TReq& request) {
+ if (request.HasCredentials()) {
+ switch (request.GetCredentials().AccessKey_case()) {
+ case TCreds::kOAuthToken: {
+ return request.GetCredentials().GetOAuthToken();
+ }
+ case TCreds::kTvmTicket: {
+ return request.GetCredentials().GetTvmTicket();
+ }
+ case TCreds::kStaticCreds: {
+ return request.GetCredentials().GetStaticCreds();
+ }
+ default: {
+ // leave the token empty
+ }
+ }
+ }
+
+ return {};
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp b/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp
index c45ec89184..3254de751c 100644
--- a/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp
+++ b/ydb/core/ymq/base/ut/dlq_helpers_ut.cpp
@@ -1,76 +1,76 @@
#include <ydb/core/ymq/base/dlq_helpers.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr::NSQS {
-
-Y_UNIT_TEST_SUITE(RedrivePolicy) {
- Y_UNIT_TEST(RedrivePolicyValidationTest) {
- NKikimrConfig::TSqsConfig config;
- {
- // basic sanity
+
+namespace NKikimr::NSQS {
+
+Y_UNIT_TEST_SUITE(RedrivePolicy) {
+ Y_UNIT_TEST(RedrivePolicyValidationTest) {
+ NKikimrConfig::TSqsConfig config;
+ {
+ // basic sanity
TRedrivePolicy basicPolicy = TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config);
- UNIT_ASSERT(basicPolicy.IsValid());
+ UNIT_ASSERT(basicPolicy.IsValid());
UNIT_ASSERT(basicPolicy.TargetArn && *basicPolicy.TargetArn == "yrn:ya:sqs:ru-central1:radix:mymegadlq");
- UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "mymegadlq");
- UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 3);
- }
- {
- // count as str
+ UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "mymegadlq");
+ UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 3);
+ }
+ {
+ // count as str
TRedrivePolicy basicPolicy = TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:ultradlq","maxReceiveCount":"42"})__", config);
- UNIT_ASSERT(basicPolicy.IsValid());
+ UNIT_ASSERT(basicPolicy.IsValid());
UNIT_ASSERT(basicPolicy.TargetArn && *basicPolicy.TargetArn == "yrn:ya:sqs:ru-central1:radix:ultradlq");
- UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "ultradlq");
- UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 42);
- }
-
+ UNIT_ASSERT(basicPolicy.TargetQueueName && *basicPolicy.TargetQueueName == "ultradlq");
+ UNIT_ASSERT(basicPolicy.MaxReceiveCount && *basicPolicy.MaxReceiveCount == 42);
+ }
+
UNIT_ASSERT(TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
- // maxReceiveCount should be from 1 to 1000
+ // maxReceiveCount should be from 1 to 1000
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":0})__", config).IsValid());
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":1001})__", config).IsValid());
- // maxReceiveCount should be integer
+ // maxReceiveCount should be integer
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":"omglol"})__", config).IsValid());
- // target arn shouldn't be empty
- UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"","maxReceiveCount":3})__", config).IsValid());
- // both fields are expected
- UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":""})__", config).IsValid());
- UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"maxReceiveCount":3})__", config).IsValid());
- // no cheats!
- UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":":","maxReceiveCount":"8"})__", config).IsValid());
- {
- // empty value is okay
- TRedrivePolicy clearPolicy = TRedrivePolicy::FromJson("", config);
- UNIT_ASSERT(clearPolicy.IsValid());
- UNIT_ASSERT(clearPolicy.TargetArn && clearPolicy.TargetArn->empty());
- UNIT_ASSERT(clearPolicy.TargetQueueName && clearPolicy.TargetQueueName->empty());
- UNIT_ASSERT(clearPolicy.MaxReceiveCount && *clearPolicy.MaxReceiveCount == 0);
- }
- }
-
- Y_UNIT_TEST(RedrivePolicyToJsonTest) {
- NKikimrConfig::TSqsConfig config;
+ // target arn shouldn't be empty
+ UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"","maxReceiveCount":3})__", config).IsValid());
+ // both fields are expected
+ UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":""})__", config).IsValid());
+ UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"maxReceiveCount":3})__", config).IsValid());
+ // no cheats!
+ UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":":","maxReceiveCount":"8"})__", config).IsValid());
+ {
+ // empty value is okay
+ TRedrivePolicy clearPolicy = TRedrivePolicy::FromJson("", config);
+ UNIT_ASSERT(clearPolicy.IsValid());
+ UNIT_ASSERT(clearPolicy.TargetArn && clearPolicy.TargetArn->empty());
+ UNIT_ASSERT(clearPolicy.TargetQueueName && clearPolicy.TargetQueueName->empty());
+ UNIT_ASSERT(clearPolicy.MaxReceiveCount && *clearPolicy.MaxReceiveCount == 0);
+ }
+ }
+
+ Y_UNIT_TEST(RedrivePolicyToJsonTest) {
+ NKikimrConfig::TSqsConfig config;
const TString json = R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__";
- UNIT_ASSERT_EQUAL(TRedrivePolicy::FromJson(json, config).ToJson(), json);
- }
-
- Y_UNIT_TEST(RedrivePolicyArnValidationTest) {
- NKikimrConfig::TSqsConfig config;
+ UNIT_ASSERT_EQUAL(TRedrivePolicy::FromJson(json, config).ToJson(), json);
+ }
+
+ Y_UNIT_TEST(RedrivePolicyArnValidationTest) {
+ NKikimrConfig::TSqsConfig config;
UNIT_ASSERT(TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
- // empty queue name
+ // empty queue name
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1:radix:","maxReceiveCount":3})__", config).IsValid());
- // empty account
+ // empty account
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central1::mymegadlq","maxReceiveCount":3})__", config).IsValid());
- // bad region
+ // bad region
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:ya:sqs:ru-central42:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
- // bad prefix
+ // bad prefix
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrrrr:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
- // broken prefix
+ // broken prefix
UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrr:ya:sqs:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
-
- config.SetYandexCloudMode(true); // check cloud prefix
+
+ config.SetYandexCloudMode(true); // check cloud prefix
UNIT_ASSERT(TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yrn:yc:ymq:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
- UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yarrr:yc:ymq:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
- }
-}
-
-} // namespace NKikimr::NSQS
+ UNIT_ASSERT(!TRedrivePolicy::FromJson(R"__({"deadLetterTargetArn":"yarrr:yc:ymq:ru-central1:radix:mymegadlq","maxReceiveCount":3})__", config).IsValid());
+ }
+}
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/ut/queue_attributes_ut.cpp b/ydb/core/ymq/base/ut/queue_attributes_ut.cpp
index aa2594131e..e33ab8ff20 100644
--- a/ydb/core/ymq/base/ut/queue_attributes_ut.cpp
+++ b/ydb/core/ymq/base/ut/queue_attributes_ut.cpp
@@ -1,48 +1,48 @@
#include <ydb/core/ymq/base/queue_attributes.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr::NSQS {
-
-Y_UNIT_TEST_SUITE(QueueAttributes) {
- Y_UNIT_TEST(BasicStdTest) {
- const bool isFifo = false;
- const bool clamp = false;
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "2.5"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "100500"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1023"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1024"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262144"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262145"}}, {}, isFifo, clamp).Validate());
- }
-
- Y_UNIT_TEST(BasicFifoTest) {
- const bool isFifo = true;
- const bool clamp = false;
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "false"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "omg"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "0"}, {"VisibilityTimeout", "1.5"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "1"}, {"VisibilityTimeout", "0"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "2"}, {"VisibilityTimeout", "31"}}, {}, isFifo, clamp).Validate());
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "3"}, {"VisibilityTimeout", "100500"}}, {}, isFifo, clamp).Validate());
- }
-
- Y_UNIT_TEST(BasicClampTest) {
- const bool isFifo = false;
- const bool clamp = true;
- UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "100000"}}, {}, isFifo, clamp).HasClampedAttributes());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).HasClampedAttributes());
- UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).HasClampedAttributes());
- UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).MaximumMessageSize != 1);
- UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).MaximumMessageSize != 1000000);
- }
-}
-
-} // namespace NKikimr::NSQS
+
+namespace NKikimr::NSQS {
+
+Y_UNIT_TEST_SUITE(QueueAttributes) {
+ Y_UNIT_TEST(BasicStdTest) {
+ const bool isFifo = false;
+ const bool clamp = false;
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "2.5"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "100500"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1023"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "1024"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262144"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"DelaySeconds", "0"}, {"MaximumMessageSize", "262145"}}, {}, isFifo, clamp).Validate());
+ }
+
+ Y_UNIT_TEST(BasicFifoTest) {
+ const bool isFifo = true;
+ const bool clamp = false;
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "true"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "false"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"FifoQueue", "omg"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "true"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ContentBasedDeduplication", "false"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "0"}, {"VisibilityTimeout", "1.5"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "1"}, {"VisibilityTimeout", "0"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "2"}, {"VisibilityTimeout", "31"}}, {}, isFifo, clamp).Validate());
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"ReceiveMessageWaitTimeSeconds", "3"}, {"VisibilityTimeout", "100500"}}, {}, isFifo, clamp).Validate());
+ }
+
+ Y_UNIT_TEST(BasicClampTest) {
+ const bool isFifo = false;
+ const bool clamp = true;
+ UNIT_ASSERT(!TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "100000"}}, {}, isFifo, clamp).HasClampedAttributes());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).HasClampedAttributes());
+ UNIT_ASSERT(TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).HasClampedAttributes());
+ UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1"}}, {}, isFifo, clamp).MaximumMessageSize != 1);
+ UNIT_ASSERT(*TQueueAttributes::FromAttributesAndConfig({{"MaximumMessageSize", "1000000"}}, {}, isFifo, clamp).MaximumMessageSize != 1000000);
+ }
+}
+
+} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/base/ut/ya.make b/ydb/core/ymq/base/ut/ya.make
index 7a9218946a..f927641a90 100644
--- a/ydb/core/ymq/base/ut/ya.make
+++ b/ydb/core/ymq/base/ut/ya.make
@@ -14,10 +14,10 @@ PEERDIR(
SRCS(
action_ut.cpp
counters_ut.cpp
- dlq_helpers_ut.cpp
+ dlq_helpers_ut.cpp
helpers_ut.cpp
secure_protobuf_printer_ut.cpp
- queue_attributes_ut.cpp
+ queue_attributes_ut.cpp
)
END()
diff --git a/ydb/core/ymq/base/ya.make b/ydb/core/ymq/base/ya.make
index 3d5ea20516..14cd4e72a1 100644
--- a/ydb/core/ymq/base/ya.make
+++ b/ydb/core/ymq/base/ya.make
@@ -6,25 +6,25 @@ OWNER(
LIBRARY()
SRCS(
- acl.cpp
+ acl.cpp
action.cpp
counters.cpp
debug_info.cpp
- dlq_helpers.cpp
+ dlq_helpers.cpp
helpers.cpp
probes.cpp
- queue_attributes.cpp
- queue_id.cpp
+ queue_attributes.cpp
+ queue_id.cpp
secure_protobuf_printer.cpp
events_writer_iface.h
)
GENERATE_ENUM_SERIALIZATION(query_id.h)
-GENERATE_ENUM_SERIALIZATION(cloud_enums.h)
+GENERATE_ENUM_SERIALIZATION(cloud_enums.h)
PEERDIR(
- contrib/libs/openssl
+ contrib/libs/openssl
library/cpp/cgiparam
library/cpp/ipmath
library/cpp/lwtrace
diff --git a/ydb/core/ymq/client/bin/main.cpp b/ydb/core/ymq/client/bin/main.cpp
index ac081949d7..cb7dbd2a2c 100644
--- a/ydb/core/ymq/client/bin/main.cpp
+++ b/ydb/core/ymq/client/bin/main.cpp
@@ -6,7 +6,7 @@
#include <util/stream/file.h>
#include <util/stream/output.h>
-#include <util/system/env.h>
+#include <util/system/env.h>
#include <util/system/user.h>
#include <google/protobuf/text_format.h>
@@ -114,7 +114,7 @@ static int HandleCreate(int argc, const char* argv[]) {
TString queue;
ui64 shards = 0;
ui64 partitions = 0;
- ui64 retentionSeconds = 0;
+ ui64 retentionSeconds = 0;
bool enableOutOfOrderExecution = false;
bool disableOutOfOrderExecution = false;
bool contentBasedDeduplication = false;
@@ -186,13 +186,13 @@ static int HandleCreate(int argc, const char* argv[]) {
req.SetEnableOutOfOrderTransactionsExecution(false);
}
opts.SetCredentials(req);
-
- if (retentionSeconds) {
- auto* newAttribute = req.mutable_attributes()->Add();
- newAttribute->SetName("MessageRetentionPeriod");
- newAttribute->SetValue(ToString(retentionSeconds));
- }
-
+
+ if (retentionSeconds) {
+ auto* newAttribute = req.mutable_attributes()->Add();
+ newAttribute->SetName("MessageRetentionPeriod");
+ newAttribute->SetValue(ToString(retentionSeconds));
+ }
+
if (enableAutosplit) {
req.SetEnableAutosplit(true); // explicitly
} else if (disableAutosplit) {
@@ -235,7 +235,7 @@ static int HandleDelete(int argc, const char* argv[]) {
req.MutableAuth()->SetUserName(opts.User);
req.SetQueueName(queue);
opts.SetCredentials(req);
-
+
auto resp = TQueueClient(TClientOptions().SetHost(opts.Host).SetPort(opts.Port)).DeleteQueue(req);
if (resp.HasError()) {
@@ -511,13 +511,13 @@ static int HandleUser(int argc, const char* argv[]) {
return 0;
}
-static int HandlePermissions(int argc, const char* argv[]) {
- TString resource;
- TString grantRequest;
- TString revokeRequest;
- TString setRequest;
- bool clearACL;
-
+static int HandlePermissions(int argc, const char* argv[]) {
+ TString resource;
+ TString grantRequest;
+ TString revokeRequest;
+ TString setRequest;
+ bool clearACL;
+
TSqsOptions opts(false);
opts.AddLongOption('q', "resource", "resource path")
.Required()
@@ -535,95 +535,95 @@ static int HandlePermissions(int argc, const char* argv[]) {
.Optional()
.NoArgument()
.SetFlag(&clearACL);
-
+
TOptsParseResult res(&opts, argc, argv);
-
+
TQueueClient client(TClientOptions().SetHost(opts.Host).SetPort(opts.Port));
-
- TModifyPermissionsRequest req;
-
- auto parseActionsAndSubject = [](auto& action, TStringBuf request) {
- auto subject = request.NextTok(':');
- if (!subject || !request) {
- return false;
- }
- while (auto actionName = request.NextTok(',')) {
- *action.MutablePermissionNames()->Add() = TString(actionName);
- }
- action.SetSubject(TString(subject));
- return true;
- };
-
- bool valid = true;
- if (grantRequest) {
- valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableGrant(), grantRequest);
- }
- if (revokeRequest) {
- valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableRevoke(), revokeRequest);
- }
- if (setRequest) {
- valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableSet(), setRequest);
- }
-
- req.SetResource(resource);
-
- if (!valid || !(grantRequest || revokeRequest || setRequest || clearACL)) {
- Cerr << "Modify permissions command is invalid. Usage example: --grant username@staff:CreateQueue,SendMessage" << Endl;
- return 1;
- }
-
+
+ TModifyPermissionsRequest req;
+
+ auto parseActionsAndSubject = [](auto& action, TStringBuf request) {
+ auto subject = request.NextTok(':');
+ if (!subject || !request) {
+ return false;
+ }
+ while (auto actionName = request.NextTok(',')) {
+ *action.MutablePermissionNames()->Add() = TString(actionName);
+ }
+ action.SetSubject(TString(subject));
+ return true;
+ };
+
+ bool valid = true;
+ if (grantRequest) {
+ valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableGrant(), grantRequest);
+ }
+ if (revokeRequest) {
+ valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableRevoke(), revokeRequest);
+ }
+ if (setRequest) {
+ valid &= parseActionsAndSubject(*req.MutableActions()->Add()->MutableSet(), setRequest);
+ }
+
+ req.SetResource(resource);
+
+ if (!valid || !(grantRequest || revokeRequest || setRequest || clearACL)) {
+ Cerr << "Modify permissions command is invalid. Usage example: --grant username@staff:CreateQueue,SendMessage" << Endl;
+ return 1;
+ }
+
opts.SetCredentials(req);
-
- if (clearACL) {
- req.SetClearACL(true);
- }
-
- auto resp = client.ModifyPermissions(req);
- if (resp.HasError()) {
- Cerr << "Got error for resource : "
- << resource << " : "
- << resp.GetError().GetMessage() << Endl;
- return 1;
- } else {
- Cerr << "OK"
- << Endl;
- }
-
- return 0;
-}
-
-static int HandleListPermissions(int argc, const char* argv[]) {
- TString path;
-
+
+ if (clearACL) {
+ req.SetClearACL(true);
+ }
+
+ auto resp = client.ModifyPermissions(req);
+ if (resp.HasError()) {
+ Cerr << "Got error for resource : "
+ << resource << " : "
+ << resp.GetError().GetMessage() << Endl;
+ return 1;
+ } else {
+ Cerr << "OK"
+ << Endl;
+ }
+
+ return 0;
+}
+
+static int HandleListPermissions(int argc, const char* argv[]) {
+ TString path;
+
TSqsOptions opts(false);
opts.AddLongOption('P', "path", "path to node")
.Required()
.StoreResult(&path);
-
+
TOptsParseResult res(&opts, argc, argv);
-
+
TQueueClient client(TClientOptions().SetHost(opts.Host).SetPort(opts.Port));
-
- TListPermissionsRequest req;
-
- req.SetPath(path);
+
+ TListPermissionsRequest req;
+
+ req.SetPath(path);
opts.SetCredentials(req);
-
- auto resp = client.ListPermissions(req);
- if (resp.HasError()) {
- Cerr << "Got error for path : "
- << path << " : "
- << resp.GetError().GetMessage() << Endl;
- return 1;
- } else {
- Cout << resp.Utf8DebugString() << Endl;
- }
-
- return 0;
-}
-
-
-
+
+ auto resp = client.ListPermissions(req);
+ if (resp.HasError()) {
+ Cerr << "Got error for path : "
+ << path << " : "
+ << resp.GetError().GetMessage() << Endl;
+ return 1;
+ } else {
+ Cout << resp.Utf8DebugString() << Endl;
+ }
+
+ return 0;
+}
+
+
+
static int HandleGetQueueAttributes(int argc, const char* argv[]) {
TString queueName;
diff --git a/ydb/core/ymq/client/cpp/client.cpp b/ydb/core/ymq/client/cpp/client.cpp
index 1814aaa9e3..c96f2dcaab 100644
--- a/ydb/core/ymq/client/cpp/client.cpp
+++ b/ydb/core/ymq/client/cpp/client.cpp
@@ -56,10 +56,10 @@ public:
METHOD_IMPL(PurgeQueue, "can't purge queue");
METHOD_IMPL(ReceiveMessage, "can't receive a message");
METHOD_IMPL(SendMessage, "can't enqueue a message");
- METHOD_IMPL(ModifyPermissions, "can't modify permissions");
+ METHOD_IMPL(ModifyPermissions, "can't modify permissions");
METHOD_IMPL(GetQueueAttributes, "can't get queue attributes");
METHOD_IMPL(SetQueueAttributes, "can't set queue attributes");
- METHOD_IMPL(ListPermissions, "can't list permissions");
+ METHOD_IMPL(ListPermissions, "can't list permissions");
#undef METHOD_IMPL
@@ -130,10 +130,10 @@ TSendMessageResponse TQueueClient::SendMessage(const TSendMessageRequest& req) {
return Impl_->SendMessage(req);
}
-TModifyPermissionsResponse TQueueClient::ModifyPermissions(const TModifyPermissionsRequest& req) {
- return Impl_->ModifyPermissions(req);
-}
-
+TModifyPermissionsResponse TQueueClient::ModifyPermissions(const TModifyPermissionsRequest& req) {
+ return Impl_->ModifyPermissions(req);
+}
+
TGetQueueAttributesResponse TQueueClient::GetQueueAttributes(const TGetQueueAttributesRequest& req) {
return Impl_->GetQueueAttributes(req);
}
@@ -142,8 +142,8 @@ TSetQueueAttributesResponse TQueueClient::SetQueueAttributes(const TSetQueueAttr
return Impl_->SetQueueAttributes(req);
}
-TListPermissionsResponse TQueueClient::ListPermissions(const TListPermissionsRequest& req) {
- return Impl_->ListPermissions(req);
-}
-
+TListPermissionsResponse TQueueClient::ListPermissions(const TListPermissionsRequest& req) {
+ return Impl_->ListPermissions(req);
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/client/cpp/client.h b/ydb/core/ymq/client/cpp/client.h
index 055431f109..d42ed84326 100644
--- a/ydb/core/ymq/client/cpp/client.h
+++ b/ydb/core/ymq/client/cpp/client.h
@@ -90,14 +90,14 @@ public:
TSendMessageResponse SendMessage(const TSendMessageRequest& req);
- TModifyPermissionsResponse ModifyPermissions(const TModifyPermissionsRequest& req);
-
+ TModifyPermissionsResponse ModifyPermissions(const TModifyPermissionsRequest& req);
+
TGetQueueAttributesResponse GetQueueAttributes(const TGetQueueAttributesRequest& req);
TSetQueueAttributesResponse SetQueueAttributes(const TSetQueueAttributesRequest& req);
- TListPermissionsResponse ListPermissions(const TListPermissionsRequest& req);
-
+ TListPermissionsResponse ListPermissions(const TListPermissionsRequest& req);
+
private:
class TImpl;
THolder<TImpl> Impl_;
diff --git a/ydb/core/ymq/http/http.cpp b/ydb/core/ymq/http/http.cpp
index 3aa9151646..5d0963bfb2 100644
--- a/ydb/core/ymq/http/http.cpp
+++ b/ydb/core/ymq/http/http.cpp
@@ -20,7 +20,7 @@
#include <util/generic/guid.h>
#include <util/generic/hash.h>
-#include <util/generic/set.h>
+#include <util/generic/set.h>
#include <util/generic/hash_set.h>
#include <util/network/init.h>
#include <util/string/ascii.h>
@@ -52,7 +52,7 @@ const TString CREDENTIAL_PARAM = "credential";
constexpr TStringBuf PRIVATE_REQUEST_PATH_PREFIX = "/private";
const TSet<TString> ModifyPermissionsActions = {"GrantPermissions", "RevokePermissions", "SetPermissions"};
-
+
bool IsPrivateTokenHeader(TStringBuf headerName) {
for (const TStringBuf h : PRIVATE_TOKENS_HEADERS) {
if (AsciiEqualsIgnoreCase(h, headerName)) {
@@ -71,14 +71,14 @@ public:
}
void DoSendReply(const TSqsResponse& resp) override {
- auto response = ResponseToAmazonXmlFormat(resp);
+ auto response = ResponseToAmazonXmlFormat(resp);
LogRequest(resp, response);
-
- response.FolderId = resp.GetFolderId();
- response.IsFifo = resp.GetIsFifo();
- response.ResourceId = resp.GetResourceId();
-
- Request_->SendResponse(response);
+
+ response.FolderId = resp.GetFolderId();
+ response.IsFifo = resp.GetIsFifo();
+ response.ResourceId = resp.GetResourceId();
+
+ Request_->SendResponse(response);
}
@@ -154,23 +154,23 @@ void THttpRequest::WriteResponse(const TReplyParams& replyParams, const TSqsHttp
}
if (Parent_->Config.GetYandexCloudMode() && !IsPrivateRequest_) {
- // Send request attributes to the metering actor
- auto reportRequestAttributes = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>();
-
- auto& requestAttributes = reportRequestAttributes->Data;
-
- requestAttributes.HttpStatusCode = response.StatusCode;
- requestAttributes.IsFifo = response.IsFifo;
- requestAttributes.FolderId = response.FolderId;
- requestAttributes.RequestSizeInBytes = RequestSizeInBytes_;
- requestAttributes.ResponseSizeInBytes = response.Body.size();
- requestAttributes.SourceAddress = SourceAddress_;
- requestAttributes.ResourceId = response.ResourceId;
- requestAttributes.Action = Action_;
-
- Parent_->ActorSystem_->Send(MakeSqsMeteringServiceID(), reportRequestAttributes.Release());
- }
-
+ // Send request attributes to the metering actor
+ auto reportRequestAttributes = MakeHolder<TSqsEvents::TEvReportProcessedRequestAttributes>();
+
+ auto& requestAttributes = reportRequestAttributes->Data;
+
+ requestAttributes.HttpStatusCode = response.StatusCode;
+ requestAttributes.IsFifo = response.IsFifo;
+ requestAttributes.FolderId = response.FolderId;
+ requestAttributes.RequestSizeInBytes = RequestSizeInBytes_;
+ requestAttributes.ResponseSizeInBytes = response.Body.size();
+ requestAttributes.SourceAddress = SourceAddress_;
+ requestAttributes.ResourceId = response.ResourceId;
+ requestAttributes.Action = Action_;
+
+ Parent_->ActorSystem_->Send(MakeSqsMeteringServiceID(), reportRequestAttributes.Release());
+ }
+
httpResponse.OutTo(replyParams.Output);
}
@@ -222,12 +222,12 @@ bool THttpRequest::DoReply(const TReplyParams& p) {
return true;
}
- try {
- ParseHeaders(p.Input);
+ try {
+ ParseHeaders(p.Input);
- if (SetupPing(p)) {
- return false;
- }
+ if (SetupPing(p)) {
+ return false;
+ }
ParseRequest(p.Input);
@@ -240,7 +240,7 @@ bool THttpRequest::DoReply(const TReplyParams& p) {
<< "] IP [" << SourceAddress_ << "]"
);
- if (!Parent_->Config.GetYandexCloudMode() && UserName_.empty()) {
+ if (!Parent_->Config.GetYandexCloudMode() && UserName_.empty()) {
WriteResponse(p, MakeErrorXmlResponse(NErrors::MISSING_PARAMETER, Parent_->AggregatedUserCounters_.Get(), "No user name was provided."));
return true;
}
@@ -277,24 +277,24 @@ TString THttpRequest::GetRequestPathPart(TStringBuf path, size_t partIdx) const
TVector<TStringBuf> items;
StringSplitter(path).Split('/').AddTo(&items);
- if (items.size() > partIdx) {
- return TString(items[partIdx]);
+ if (items.size() > partIdx) {
+ return TString(items[partIdx]);
}
return TString();
}
TString THttpRequest::ExtractQueueNameFromPath(const TStringBuf path) {
- return GetRequestPathPart(path, 2);
-}
-
+ return GetRequestPathPart(path, 2);
+}
+
TString THttpRequest::ExtractAccountNameFromPath(const TStringBuf path) {
- return GetRequestPathPart(path, 1);
-}
-
+ return GetRequestPathPart(path, 1);
+}
+
void THttpRequest::ExtractQueueAndAccountNames(const TStringBuf path) {
- if (Action_ == EAction::ModifyPermissions)
- return;
-
+ if (Action_ == EAction::ModifyPermissions)
+ return;
+
if (Action_ == EAction::GetQueueUrl || Action_ == EAction::CreateQueue) {
if (!QueryParams_.QueueName) {
throw TSQSException(NErrors::MISSING_PARAMETER) << "No queue name was provided.";
@@ -302,9 +302,9 @@ void THttpRequest::ExtractQueueAndAccountNames(const TStringBuf path) {
QueueName_ = *QueryParams_.QueueName;
} else {
- const auto pathAndQuery = QueryParams_.QueueUrl ? GetPathAndQuery(*QueryParams_.QueueUrl) : GetPathAndQuery(path);
- QueueName_ = ExtractQueueNameFromPath(pathAndQuery);
- AccountName_ = ExtractAccountNameFromPath(pathAndQuery);
+ const auto pathAndQuery = QueryParams_.QueueUrl ? GetPathAndQuery(*QueryParams_.QueueUrl) : GetPathAndQuery(path);
+ QueueName_ = ExtractQueueNameFromPath(pathAndQuery);
+ AccountName_ = ExtractAccountNameFromPath(pathAndQuery);
if (IsProxyAction(Action_)) {
if (QueryParams_.QueueUrl && *QueryParams_.QueueUrl) {
@@ -346,10 +346,10 @@ void THttpRequest::ParseHeaders(const THttpInput& input) {
for (const auto& header : input.Headers()) {
if (AsciiEqualsIgnoreCase(header.Name(), AUTHORIZATION_HEADER)) {
ParseAuthorization(header.Value());
- } else if (AsciiEqualsIgnoreCase(header.Name(), SECURITY_TOKEN_HEADER)) {
- SecurityToken_ = header.Value();
- } else if (AsciiEqualsIgnoreCase(header.Name(), IAM_TOKEN_HEADER)) {
- IamToken_ = header.Value();
+ } else if (AsciiEqualsIgnoreCase(header.Name(), SECURITY_TOKEN_HEADER)) {
+ SecurityToken_ = header.Value();
+ } else if (AsciiEqualsIgnoreCase(header.Name(), IAM_TOKEN_HEADER)) {
+ IamToken_ = header.Value();
} else if (AsciiEqualsIgnoreCase(header.Name(), FORWARDED_IP_HEADER)) {
SourceAddress_ = header.Value();
} else if (AsciiEqualsIgnoreCase(header.Name(), REQUEST_ID_HEADER)) {
@@ -392,20 +392,20 @@ void THttpRequest::ParsePrivateRequestPathPrefix(const TStringBuf& path) {
}
}
-ui64 THttpRequest::CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const {
- ui64 requestSize = input.FirstLine().size();
- for (const auto& header : input.Headers()) {
- requestSize += header.Name().size() + header.Value().size();
- }
- if (input.Trailers()) {
- for (const auto& header : *input.Trailers()) {
- requestSize += header.Name().size() + header.Value().size();
- }
- }
- requestSize += contentLength;
- return requestSize;
-}
-
+ui64 THttpRequest::CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const {
+ ui64 requestSize = input.FirstLine().size();
+ for (const auto& header : input.Headers()) {
+ requestSize += header.Name().size() + header.Value().size();
+ }
+ if (input.Trailers()) {
+ for (const auto& header : *input.Trailers()) {
+ requestSize += header.Name().size() + header.Value().size();
+ }
+ }
+ requestSize += contentLength;
+ return requestSize;
+}
+
void THttpRequest::ParseRequest(THttpInput& input) {
if (Parent_->HttpCounters_ && UserName_) {
UserCounters_ = Parent_->HttpCounters_->GetUserCounters(UserName_);
@@ -433,9 +433,9 @@ void THttpRequest::ParseRequest(THttpInput& input) {
RLOG_SQS_BASE_DEBUG(*Parent_->ActorSystem_, "Incoming http request: " << input.FirstLine());
ParsePrivateRequestPathPrefix(parsed.Path);
-
- RequestSizeInBytes_ = CalculateRequestSizeInBytes(input, contentLength);
-
+
+ RequestSizeInBytes_ = CalculateRequestSizeInBytes(input, contentLength);
+
if (HttpMethod == "POST") {
ParseCgiParameters(TCgiParameters(TStringBuf(InputData->Data(), contentLength)));
} else if (HttpMethod == "GET") {
@@ -445,52 +445,52 @@ void THttpRequest::ParseRequest(THttpInput& input) {
}
if (QueryParams_.Action) {
- if (IsIn(ModifyPermissionsActions, *QueryParams_.Action)) {
- Action_ = EAction::ModifyPermissions;
- } else {
- Action_ = ActionFromString(*QueryParams_.Action);
- }
-
+ if (IsIn(ModifyPermissionsActions, *QueryParams_.Action)) {
+ Action_ = EAction::ModifyPermissions;
+ } else {
+ Action_ = ActionFromString(*QueryParams_.Action);
+ }
+
THttpActionCounters* counters = GetActionCounters();
INC_COUNTER(counters, Requests);
} else {
throw TSQSException(NErrors::MISSING_ACTION) << "Action param was not found.";
}
- if (QueryParams_.FolderId) {
- FolderId_ = *QueryParams_.FolderId;
- }
-
- ExtractQueueAndAccountNames(parsed.Path);
-
- if (Parent_->Config.GetYandexCloudMode() && !IamToken_ && !FolderId_) {
- AwsSignature_.Reset(new TAwsRequestSignV4(input, parsed, InputData));
- }
+ if (QueryParams_.FolderId) {
+ FolderId_ = *QueryParams_.FolderId;
+ }
+
+ ExtractQueueAndAccountNames(parsed.Path);
+
+ if (Parent_->Config.GetYandexCloudMode() && !IamToken_ && !FolderId_) {
+ AwsSignature_.Reset(new TAwsRequestSignV4(input, parsed, InputData));
+ }
}
-#define HANDLE_SETUP_ACTION_CASE(NAME) \
- case EAction::NAME: { \
- Y_CAT(Setup, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \
- CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \
- break; \
+#define HANDLE_SETUP_ACTION_CASE(NAME) \
+ case EAction::NAME: { \
+ Y_CAT(Setup, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \
+ CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \
+ break; \
}
-
-#define HANDLE_SETUP_PRIVATE_ACTION_CASE(NAME) \
- case EAction::NAME: { \
- if (!IsPrivateRequest_) { \
+
+#define HANDLE_SETUP_PRIVATE_ACTION_CASE(NAME) \
+ case EAction::NAME: { \
+ if (!IsPrivateRequest_) { \
RLOG_SQS_BASE_ERROR(*Parent_->ActorSystem_, \
- "Attempt to call " \
- Y_STRINGIZE(NAME) \
- " action without private url path"); \
- throw TSQSException(NErrors::INVALID_ACTION); \
- } \
- Y_CAT(SetupPrivate, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \
- CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \
- break; \
+ "Attempt to call " \
+ Y_STRINGIZE(NAME) \
+ " action without private url path"); \
+ throw TSQSException(NErrors::INVALID_ACTION); \
+ } \
+ Y_CAT(SetupPrivate, NAME)(requestHolder->Y_CAT(Mutable, NAME)()); \
+ CopyCredentials(requestHolder->Y_CAT(Mutable, NAME)(), Parent_->Config); \
+ break; \
}
bool THttpRequest::SetupRequest() {
- auto requestHolder = MakeHolder<TSqsRequest>();
+ auto requestHolder = MakeHolder<TSqsRequest>();
requestHolder->SetRequestId(RequestId_);
// Validate batches
@@ -514,29 +514,29 @@ bool THttpRequest::SetupRequest() {
}
switch (Action_) {
- HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibility);
- HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibilityBatch);
- HANDLE_SETUP_ACTION_CASE(CreateQueue);
- HANDLE_SETUP_ACTION_CASE(CreateUser);
- HANDLE_SETUP_ACTION_CASE(GetQueueAttributes);
- HANDLE_SETUP_ACTION_CASE(GetQueueUrl);
- HANDLE_SETUP_ACTION_CASE(DeleteMessage);
- HANDLE_SETUP_ACTION_CASE(DeleteMessageBatch);
- HANDLE_SETUP_ACTION_CASE(DeleteQueue);
- HANDLE_SETUP_ACTION_CASE(DeleteUser);
- HANDLE_SETUP_ACTION_CASE(ListPermissions);
- HANDLE_SETUP_ACTION_CASE(ListDeadLetterSourceQueues);
- HANDLE_SETUP_ACTION_CASE(ListQueues);
- HANDLE_SETUP_ACTION_CASE(ListUsers);
- HANDLE_SETUP_ACTION_CASE(ModifyPermissions);
- HANDLE_SETUP_ACTION_CASE(PurgeQueue);
- HANDLE_SETUP_ACTION_CASE(ReceiveMessage);
- HANDLE_SETUP_ACTION_CASE(SendMessage);
- HANDLE_SETUP_ACTION_CASE(SendMessageBatch);
- HANDLE_SETUP_ACTION_CASE(SetQueueAttributes);
+ HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibility);
+ HANDLE_SETUP_ACTION_CASE(ChangeMessageVisibilityBatch);
+ HANDLE_SETUP_ACTION_CASE(CreateQueue);
+ HANDLE_SETUP_ACTION_CASE(CreateUser);
+ HANDLE_SETUP_ACTION_CASE(GetQueueAttributes);
+ HANDLE_SETUP_ACTION_CASE(GetQueueUrl);
+ HANDLE_SETUP_ACTION_CASE(DeleteMessage);
+ HANDLE_SETUP_ACTION_CASE(DeleteMessageBatch);
+ HANDLE_SETUP_ACTION_CASE(DeleteQueue);
+ HANDLE_SETUP_ACTION_CASE(DeleteUser);
+ HANDLE_SETUP_ACTION_CASE(ListPermissions);
+ HANDLE_SETUP_ACTION_CASE(ListDeadLetterSourceQueues);
+ HANDLE_SETUP_ACTION_CASE(ListQueues);
+ HANDLE_SETUP_ACTION_CASE(ListUsers);
+ HANDLE_SETUP_ACTION_CASE(ModifyPermissions);
+ HANDLE_SETUP_ACTION_CASE(PurgeQueue);
+ HANDLE_SETUP_ACTION_CASE(ReceiveMessage);
+ HANDLE_SETUP_ACTION_CASE(SendMessage);
+ HANDLE_SETUP_ACTION_CASE(SendMessageBatch);
+ HANDLE_SETUP_ACTION_CASE(SetQueueAttributes);
HANDLE_SETUP_PRIVATE_ACTION_CASE(DeleteQueueBatch);
- HANDLE_SETUP_PRIVATE_ACTION_CASE(CountQueues);
+ HANDLE_SETUP_PRIVATE_ACTION_CASE(CountQueues);
HANDLE_SETUP_PRIVATE_ACTION_CASE(PurgeQueueBatch);
HANDLE_SETUP_PRIVATE_ACTION_CASE(GetQueueAttributesBatch);
@@ -547,7 +547,7 @@ bool THttpRequest::SetupRequest() {
}
RLOG_SQS_BASE_DEBUG(*Parent_->ActorSystem_, "Create proxy action actor for request " << SecureShortUtf8DebugString(*requestHolder));
-
+
const bool enableQueueLeader = Parent_->Config.HasEnableQueueMaster()
? Parent_->Config.GetEnableQueueMaster()
: Parent_->Config.GetEnableQueueLeader();
@@ -656,16 +656,16 @@ void THttpRequest::SetupDeleteQueue(TDeleteQueueRequest* const req) {
}
void THttpRequest::SetupListPermissions(TListPermissionsRequest* const req) {
- if (QueryParams_.Path) {
- req->SetPath(*QueryParams_.Path);
- }
-}
-
-void THttpRequest::SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req) {
- req->SetQueueName(QueueName_);
- req->MutableAuth()->SetUserName(UserName_);
-}
-
+ if (QueryParams_.Path) {
+ req->SetPath(*QueryParams_.Path);
+ }
+}
+
+void THttpRequest::SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req) {
+ req->SetQueueName(QueueName_);
+ req->MutableAuth()->SetUserName(UserName_);
+}
+
void THttpRequest::SetupPrivateDeleteQueueBatch(TDeleteQueueBatchRequest* const req) {
for (const auto& entry : QueryParams_.BatchEntries) {
auto* protoEntry = req->AddEntries();
@@ -680,10 +680,10 @@ void THttpRequest::SetupPrivateDeleteQueueBatch(TDeleteQueueBatchRequest* const
req->MutableAuth()->SetUserName(UserName_);
}
-void THttpRequest::SetupPrivateCountQueues(TCountQueuesRequest* const req) {
- req->MutableAuth()->SetUserName(UserName_);
-}
-
+void THttpRequest::SetupPrivateCountQueues(TCountQueuesRequest* const req) {
+ req->MutableAuth()->SetUserName(UserName_);
+}
+
void THttpRequest::SetupPrivatePurgeQueueBatch(TPurgeQueueBatchRequest* const req) {
for (const auto& entry : QueryParams_.BatchEntries) {
auto* protoEntry = req->AddEntries();
@@ -753,43 +753,43 @@ void THttpRequest::SetupListUsers(TListUsersRequest* const req) {
}
}
-template<typename TModifyPermissionsAction>
-static void SetupModifyPermissionsAction(const TParameters& queryParams, TModifyPermissionsAction& action) {
- if (queryParams.Subject) {
- action.SetSubject(*queryParams.Subject);
- }
-
- for (const auto& [index, entry] : queryParams.BatchEntries) {
- if (entry.Action) {
- *action.MutablePermissionNames()->Add() = *entry.Action;
- }
- }
-}
-
+template<typename TModifyPermissionsAction>
+static void SetupModifyPermissionsAction(const TParameters& queryParams, TModifyPermissionsAction& action) {
+ if (queryParams.Subject) {
+ action.SetSubject(*queryParams.Subject);
+ }
+
+ for (const auto& [index, entry] : queryParams.BatchEntries) {
+ if (entry.Action) {
+ *action.MutablePermissionNames()->Add() = *entry.Action;
+ }
+ }
+}
+
void THttpRequest::SetupModifyPermissions(TModifyPermissionsRequest* const req) {
- auto it = ModifyPermissionsActions.begin();
- if (*QueryParams_.Action == *it++) {
- SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableGrant());
- } else if (*QueryParams_.Action == *it++) {
- SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableRevoke());
- } else if (*QueryParams_.Action == *it) {
- SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableSet());
- }
-
- Y_VERIFY(it != ModifyPermissionsActions.end());
-
- if (QueryParams_.Path) {
- req->SetResource(*QueryParams_.Path);
- }
-
- if (QueryParams_.Clear) {
- bool clear = false;
- if (TryFromString(*QueryParams_.Clear, clear)) {
- req->SetClearACL(clear);
- }
- }
-}
-
+ auto it = ModifyPermissionsActions.begin();
+ if (*QueryParams_.Action == *it++) {
+ SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableGrant());
+ } else if (*QueryParams_.Action == *it++) {
+ SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableRevoke());
+ } else if (*QueryParams_.Action == *it) {
+ SetupModifyPermissionsAction(QueryParams_, *req->MutableActions()->Add()->MutableSet());
+ }
+
+ Y_VERIFY(it != ModifyPermissionsActions.end());
+
+ if (QueryParams_.Path) {
+ req->SetResource(*QueryParams_.Path);
+ }
+
+ if (QueryParams_.Clear) {
+ bool clear = false;
+ if (TryFromString(*QueryParams_.Clear, clear)) {
+ req->SetClearACL(clear);
+ }
+ }
+}
+
void THttpRequest::SetupPurgeQueue(TPurgeQueueRequest* const req) {
req->SetQueueName(QueueName_);
req->MutableAuth()->SetUserName(UserName_);
diff --git a/ydb/core/ymq/http/http.h b/ydb/core/ymq/http/http.h
index d0f6e15cfd..01a2b0edc4 100644
--- a/ydb/core/ymq/http/http.h
+++ b/ydb/core/ymq/http/http.h
@@ -51,14 +51,14 @@ private:
if (SecurityToken_ && !config.GetYandexCloudMode()) {
// it's also TVM-compatible due to universal TicketParser
request->MutableCredentials()->SetOAuthToken(SecurityToken_);
- }
+ }
}
-
+
TString GetRequestPathPart(TStringBuf path, size_t partIdx) const;
TString ExtractQueueNameFromPath(const TStringBuf path);
TString ExtractAccountNameFromPath(const TStringBuf path);
- ui64 CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const;
+ ui64 CalculateRequestSizeInBytes(const THttpInput& input, const ui64 contentLength) const;
void ExtractQueueAndAccountNames(const TStringBuf path);
TString HttpHeadersLogString(const THttpInput& input);
@@ -80,13 +80,13 @@ private:
void SetupDeleteMessageBatch(TDeleteMessageBatchRequest* const req);
void SetupDeleteQueue(TDeleteQueueRequest* const req);
void SetupListPermissions(TListPermissionsRequest* const req);
- void SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req);
+ void SetupListDeadLetterSourceQueues(TListDeadLetterSourceQueuesRequest* const req);
void SetupPrivateDeleteQueueBatch(TDeleteQueueBatchRequest* const req);
void SetupPrivatePurgeQueueBatch(TPurgeQueueBatchRequest* const req);
void SetupPrivateGetQueueAttributesBatch(TGetQueueAttributesBatchRequest* const req);
void SetupDeleteUser(TDeleteUserRequest* const req);
void SetupListQueues(TListQueuesRequest* const req);
- void SetupPrivateCountQueues(TCountQueuesRequest* const req);
+ void SetupPrivateCountQueues(TCountQueuesRequest* const req);
void SetupListUsers(TListUsersRequest* const req);
void SetupModifyPermissions(TModifyPermissionsRequest* const req);
void SetupReceiveMessage(TReceiveMessageRequest* const req);
@@ -120,7 +120,7 @@ private:
TString ApiMethod_;
THolder<TAwsRequestSignV4> AwsSignature_;
-
+
TMaybe<TBuffer> InputData;
TString HttpMethod;
TMaybe<TSqsHttpResponse> Response_;
@@ -129,8 +129,8 @@ private:
// Source values parsed from headers
TString SourceAddress_;
- ui64 RequestSizeInBytes_ = 0;
-
+ ui64 RequestSizeInBytes_ = 0;
+
bool IsPrivateRequest_ = false; // Has "/private" path prefix
TInstant StartTime_ = TInstant::Now();
};
@@ -169,7 +169,7 @@ private:
const NKikimrConfig::TSqsConfig Config;
NActors::TActorSystem* ActorSystem_ = nullptr;
TIntrusivePtr<THttpCounters> HttpCounters_; // http subsystem counters
- THolder<TCloudAuthCounters> CloudAuthCounters_; // cloud_auth subsystem counters
+ THolder<TCloudAuthCounters> CloudAuthCounters_; // cloud_auth subsystem counters
TIntrusivePtr<TUserCounters> AggregatedUserCounters_; // aggregated counters for user in core subsystem
ui32 PoolId_ = 0;
};
diff --git a/ydb/core/ymq/http/params.h b/ydb/core/ymq/http/params.h
index 601e0e7f94..7834fc4f0f 100644
--- a/ydb/core/ymq/http/params.h
+++ b/ydb/core/ymq/http/params.h
@@ -10,21 +10,21 @@ namespace NKikimr::NSQS {
struct TParameters {
TMaybe<TString> Action;
- TMaybe<TString> Clear;
+ TMaybe<TString> Clear;
TMaybe<ui64> DelaySeconds;
- TMaybe<TString> FolderId;
+ TMaybe<TString> FolderId;
TMaybe<TString> Id;
TMaybe<ui32> MaxNumberOfMessages;
TMaybe<TString> MessageBody;
TMaybe<TString> MessageDeduplicationId;
TMaybe<TString> MessageGroupId;
- TMaybe<TString> Path;
+ TMaybe<TString> Path;
TMaybe<TString> QueueName;
TMaybe<TString> QueueNamePrefix;
TMaybe<TString> QueueUrl;
TMaybe<TString> ReceiptHandle;
TMaybe<TString> ReceiveRequestAttemptId;
- TMaybe<TString> Subject;
+ TMaybe<TString> Subject;
TMaybe<TString> UserName;
TMaybe<TString> UserNamePrefix;
TMaybe<TString> Version;
diff --git a/ydb/core/ymq/http/parser.rl6 b/ydb/core/ymq/http/parser.rl6
index 6db93b4a1f..b617c334a3 100644
--- a/ydb/core/ymq/http/parser.rl6
+++ b/ydb/core/ymq/http/parser.rl6
@@ -112,28 +112,28 @@ get_queue_attributes_entry =
(("Id" % { CurrentParams_->Id = value; }) |
("QueueUrl" % { CurrentParams_->QueueUrl = value; }));
-permissions_entry =
- "Permission" > { Id_ = 1; } ('.' int %{ Id_ = I; CurrentParams_ = &Params_->BatchEntries[Id_]; CurrentParams_->Action = value; });
-
+permissions_entry =
+ "Permission" > { Id_ = 1; } ('.' int %{ Id_ = I; CurrentParams_ = &Params_->BatchEntries[Id_]; CurrentParams_->Action = value; });
+
main := |*
('Action') { CurrentParams_->Action = value; };
- ('Clear') { CurrentParams_->Clear = value; };
+ ('Clear') { CurrentParams_->Clear = value; };
('DelaySeconds') { CurrentParams_->DelaySeconds = ParseAndValidate(value, TStringBuf("DelaySeconds")); };
- ('folderId') { CurrentParams_->FolderId = value; };
+ ('folderId') { CurrentParams_->FolderId = value; };
('MaxNumberOfMessages') { CurrentParams_->MaxNumberOfMessages = ParseAndValidate(value, TStringBuf("MaxNumberOfMessages")); };
('MessageBody') { CurrentParams_->MessageBody = value; };
('MessageDeduplicationId') { CurrentParams_->MessageDeduplicationId = ValidateAlphaNumAndPunctuation128ForAssign(value, TStringBuf("MessageDeduplicationId")); };
('MessageGroupId') { CurrentParams_->MessageGroupId = ValidateAlphaNumAndPunctuation128ForAssign(value, TStringBuf("MessageGroupId")); };
- ('Path') { CurrentParams_->Path = value; };
+ ('Path') { CurrentParams_->Path = value; };
('QueueName') { CurrentParams_->QueueName = value; };
('QueueNamePrefix') { CurrentParams_->QueueNamePrefix = value; };
('QueueUrl') { CurrentParams_->QueueUrl = value; };
('ReceiptHandle') { CurrentParams_->ReceiptHandle = value; };
('ReceiveRequestAttemptId') { CurrentParams_->ReceiveRequestAttemptId = ValidateAlphaNumAndPunctuation128ForAssign(value, TStringBuf("ReceiveRequestAttemptId")); };
- ('Subject') { CurrentParams_->Subject = value; };
+ ('Subject') { CurrentParams_->Subject = value; };
('UserName') { CurrentParams_->UserName = value; };
('UserNamePrefix') { CurrentParams_->UserNamePrefix = value; };
- ('Version') { CurrentParams_->Version = value; };
+ ('Version') { CurrentParams_->Version = value; };
('VisibilityTimeout') { CurrentParams_->VisibilityTimeout = ParseAndValidate(value, TStringBuf("VisibilityTimeout")); };
('WaitTimeSeconds') { CurrentParams_->WaitTimeSeconds = ParseAndValidate(value, TStringBuf("WaitTimeSeconds")); };
@@ -142,7 +142,7 @@ main := |*
change_visibility_entry;
delete_message_entry;
message_attribute;
- permissions_entry;
+ permissions_entry;
send_message_entry;
delete_queue_entry;
purge_queue_entry;
diff --git a/ydb/core/ymq/http/types.h b/ydb/core/ymq/http/types.h
index fdf98e4b02..c33574572e 100644
--- a/ydb/core/ymq/http/types.h
+++ b/ydb/core/ymq/http/types.h
@@ -14,10 +14,10 @@ struct TSqsHttpResponse {
TString ContentType;
int StatusCode = 0;
- TString FolderId;
- TString ResourceId;
+ TString FolderId;
+ TString ResourceId;
bool IsFifo = false;
-
+
TSqsHttpResponse() = default;
TSqsHttpResponse(const TString& body, int status, const TString& contentType = XML_CONTENT_TYPE);
};
diff --git a/ydb/core/ymq/http/xml.cpp b/ydb/core/ymq/http/xml.cpp
index d589cc9a32..0cb6f85274 100644
--- a/ydb/core/ymq/http/xml.cpp
+++ b/ydb/core/ymq/http/xml.cpp
@@ -69,10 +69,10 @@ static bool MaybeErrorResponse(const T& resp, TStringBuilder& builder) {
return false;
}
-static TString BoolToString(const bool b) {
- return TString(b ? "true" : "false");
-}
-
+static TString BoolToString(const bool b) {
+ return TString(b ? "true" : "false");
+}
+
void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStringBuilder& xmlBuilder) {
if (rec.HasApproximateNumberOfMessages()) {
XML_ELEM("Attribute") {
@@ -95,7 +95,7 @@ void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStrin
if (rec.HasContentBasedDeduplication()) {
XML_ELEM("Attribute") {
XML_ELEM_CONT("Name", "ContentBasedDeduplication");
- XML_ELEM_CONT("Value", BoolToString(rec.GetContentBasedDeduplication()));
+ XML_ELEM_CONT("Value", BoolToString(rec.GetContentBasedDeduplication()));
}
}
if (rec.HasCreatedTimestamp()) {
@@ -113,7 +113,7 @@ void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStrin
if (rec.HasFifoQueue()) {
XML_ELEM("Attribute") {
XML_ELEM_CONT("Name", "FifoQueue");
- XML_ELEM_CONT("Value", BoolToString(rec.GetFifoQueue()));
+ XML_ELEM_CONT("Value", BoolToString(rec.GetFifoQueue()));
}
}
if (rec.HasMaximumMessageSize()) {
@@ -140,18 +140,18 @@ void WriteQueueAttributesToXml(const TGetQueueAttributesResponse& rec, TXmlStrin
XML_ELEM_CONT("Value", ToString(rec.GetVisibilityTimeout()));
}
}
- if (rec.HasRedrivePolicy()) {
- XML_ELEM("Attribute") {
- XML_ELEM_CONT("Name", "RedrivePolicy");
- XML_ELEM_CONT("Value", ToString(rec.GetRedrivePolicy()));
- }
- }
- if (rec.HasQueueArn()) {
- XML_ELEM("Attribute") {
- XML_ELEM_CONT("Name", "QueueArn");
- XML_ELEM_CONT("Value", ToString(rec.GetQueueArn()));
- }
- }
+ if (rec.HasRedrivePolicy()) {
+ XML_ELEM("Attribute") {
+ XML_ELEM_CONT("Name", "RedrivePolicy");
+ XML_ELEM_CONT("Value", ToString(rec.GetRedrivePolicy()));
+ }
+ }
+ if (rec.HasQueueArn()) {
+ XML_ELEM("Attribute") {
+ XML_ELEM_CONT("Name", "QueueArn");
+ XML_ELEM_CONT("Value", ToString(rec.GetQueueArn()));
+ }
+ }
}
TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) {
@@ -409,24 +409,24 @@ TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) {
break;
}
- case TSqsResponse::kCountQueues: {
+ case TSqsResponse::kCountQueues: {
HANDLE_ERROR(CountQueues);
- XML_BUILDER() {
- XML_DOC() {
- XML_ELEM("CountQueuesResponse") {
- XML_ELEM("CountQueuesResult") {
- XML_ELEM_CONT("Count", ::ToString(resp.GetCountQueues().GetCount()));
- }
- XML_ELEM("ResponseMetadata") {
- XML_ELEM_CONT("RequestId", resp.GetCountQueues().GetRequestId());
- }
- }
- }
- }
- result << XML_RESULT();
- break;
- }
-
+ XML_BUILDER() {
+ XML_DOC() {
+ XML_ELEM("CountQueuesResponse") {
+ XML_ELEM("CountQueuesResult") {
+ XML_ELEM_CONT("Count", ::ToString(resp.GetCountQueues().GetCount()));
+ }
+ XML_ELEM("ResponseMetadata") {
+ XML_ELEM_CONT("RequestId", resp.GetCountQueues().GetRequestId());
+ }
+ }
+ }
+ }
+ result << XML_RESULT();
+ break;
+ }
+
case TSqsResponse::kListUsers: {
HANDLE_ERROR(ListUsers);
XML_BUILDER() {
@@ -535,9 +535,9 @@ TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) {
if (message.HasSentTimestamp()) {
ATTRIBUTE("SentTimestamp", message.GetSentTimestamp());
}
- if (message.HasSenderId()) {
- ATTRIBUTE("SenderId", message.GetSenderId());
- }
+ if (message.HasSenderId()) {
+ ATTRIBUTE("SenderId", message.GetSenderId());
+ }
// message attributes
for (const auto& attr : message.messageattributes()) {
@@ -679,83 +679,83 @@ TSqsHttpResponse ResponseToAmazonXmlFormat(const TSqsResponse& resp) {
break;
}
- case TSqsResponse::kModifyPermissions: {
+ case TSqsResponse::kModifyPermissions: {
HANDLE_ERROR(ModifyPermissions);
- XML_BUILDER() {
- XML_DOC() {
- XML_ELEM("ModifyPermissionsResponse") {
- XML_ELEM("ResponseMetadata") {
- XML_ELEM_CONT("RequestId", resp.GetModifyPermissions().GetRequestId());
- }
- }
- }
- }
- result << XML_RESULT();
- break;
- }
-
- case TSqsResponse::kListPermissions: {
+ XML_BUILDER() {
+ XML_DOC() {
+ XML_ELEM("ModifyPermissionsResponse") {
+ XML_ELEM("ResponseMetadata") {
+ XML_ELEM_CONT("RequestId", resp.GetModifyPermissions().GetRequestId());
+ }
+ }
+ }
+ }
+ result << XML_RESULT();
+ break;
+ }
+
+ case TSqsResponse::kListPermissions: {
HANDLE_ERROR(ListPermissions);
- const auto& listPermissions = resp.GetListPermissions();
-
+ const auto& listPermissions = resp.GetListPermissions();
+
#define SERIALIZE_PERMISSIONS(resource, permission) \
- for (size_t i = 0; i < listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Y_CAT(permission, s), Size)(); ++i) { \
- XML_ELEM_IMPL("Ya" Y_STRINGIZE(permission), Y_CAT(__LINE__, a)) { \
- const auto& permissions = listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Get, Y_CAT(permission, s))(i); \
- XML_ELEM_CONT_IMPL("Subject", permissions.GetSubject(), Y_CAT(__LINE__, b)); \
- for (size_t j = 0; j < permissions.PermissionNamesSize(); ++j) { \
- XML_ELEM_CONT_IMPL("Permission", permissions.GetPermissionNames(j), Y_CAT(__LINE__, c)); \
- } \
- } \
- }
-
-#define SERIALIZE_PERMISSIONS_FOR_RESOURCE(resource) \
- SERIALIZE_PERMISSIONS(resource, EffectivePermission); \
- SERIALIZE_PERMISSIONS(resource, Permission); \
- XML_ELEM_CONT("ResourceType", Y_STRINGIZE(resource));
-
- XML_BUILDER() {
- XML_DOC() {
- XML_ELEM("ListPermissionsResponse") {
- XML_ELEM("YaListPermissionsResult") {
- if (listPermissions.HasQueuePermissions()) {
- SERIALIZE_PERMISSIONS_FOR_RESOURCE(Queue);
- } else if (listPermissions.HasAccountPermissions()) {
- SERIALIZE_PERMISSIONS_FOR_RESOURCE(Account);
- }
- XML_ELEM("ResponseMetadata") {
- XML_ELEM_CONT("RequestId", listPermissions.GetRequestId());
- }
- }
- }
- }
- }
- result << XML_RESULT();
- break;
- }
-#undef SERIALIZE_PERMISSIONS_FOR_RESOURCE
-#undef SERIALIZE_PERMISSIONS
- case TSqsResponse::kListDeadLetterSourceQueues: {
+ for (size_t i = 0; i < listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Y_CAT(permission, s), Size)(); ++i) { \
+ XML_ELEM_IMPL("Ya" Y_STRINGIZE(permission), Y_CAT(__LINE__, a)) { \
+ const auto& permissions = listPermissions.Y_CAT(Get, Y_CAT(resource, Permissions))().Y_CAT(Get, Y_CAT(permission, s))(i); \
+ XML_ELEM_CONT_IMPL("Subject", permissions.GetSubject(), Y_CAT(__LINE__, b)); \
+ for (size_t j = 0; j < permissions.PermissionNamesSize(); ++j) { \
+ XML_ELEM_CONT_IMPL("Permission", permissions.GetPermissionNames(j), Y_CAT(__LINE__, c)); \
+ } \
+ } \
+ }
+
+#define SERIALIZE_PERMISSIONS_FOR_RESOURCE(resource) \
+ SERIALIZE_PERMISSIONS(resource, EffectivePermission); \
+ SERIALIZE_PERMISSIONS(resource, Permission); \
+ XML_ELEM_CONT("ResourceType", Y_STRINGIZE(resource));
+
+ XML_BUILDER() {
+ XML_DOC() {
+ XML_ELEM("ListPermissionsResponse") {
+ XML_ELEM("YaListPermissionsResult") {
+ if (listPermissions.HasQueuePermissions()) {
+ SERIALIZE_PERMISSIONS_FOR_RESOURCE(Queue);
+ } else if (listPermissions.HasAccountPermissions()) {
+ SERIALIZE_PERMISSIONS_FOR_RESOURCE(Account);
+ }
+ XML_ELEM("ResponseMetadata") {
+ XML_ELEM_CONT("RequestId", listPermissions.GetRequestId());
+ }
+ }
+ }
+ }
+ }
+ result << XML_RESULT();
+ break;
+ }
+#undef SERIALIZE_PERMISSIONS_FOR_RESOURCE
+#undef SERIALIZE_PERMISSIONS
+ case TSqsResponse::kListDeadLetterSourceQueues: {
HANDLE_ERROR(ListDeadLetterSourceQueues);
- XML_BUILDER() {
- XML_DOC() {
- XML_ELEM("ListDeadLetterSourceQueuesResponse") {
- XML_ELEM("ListDeadLetterSourceQueuesResult") {
- for (const auto& item : resp.GetListDeadLetterSourceQueues().queues()) {
- XML_ELEM_CONT("QueueUrl", item.GetQueueUrl());
- }
- }
- XML_ELEM("ResponseMetadata") {
- XML_ELEM_CONT("RequestId", resp.GetListDeadLetterSourceQueues().GetRequestId());
- }
- }
- }
- }
- result << XML_RESULT();
- break;
- }
-
+ XML_BUILDER() {
+ XML_DOC() {
+ XML_ELEM("ListDeadLetterSourceQueuesResponse") {
+ XML_ELEM("ListDeadLetterSourceQueuesResult") {
+ for (const auto& item : resp.GetListDeadLetterSourceQueues().queues()) {
+ XML_ELEM_CONT("QueueUrl", item.GetQueueUrl());
+ }
+ }
+ XML_ELEM("ResponseMetadata") {
+ XML_ELEM_CONT("RequestId", resp.GetListDeadLetterSourceQueues().GetRequestId());
+ }
+ }
+ }
+ }
+ result << XML_RESULT();
+ break;
+ }
+
case TSqsResponse::RESPONSE_NOT_SET: {
return MakeErrorXmlResponse(NErrors::INTERNAL_FAILURE, nullptr, "Not implemented.");
}
diff --git a/ydb/core/ymq/queues/fifo/queries.cpp b/ydb/core/ymq/queues/fifo/queries.cpp
index e43ac55f2e..5876be81d6 100644
--- a/ydb/core/ymq/queues/fifo/queries.cpp
+++ b/ydb/core/ymq/queues/fifo/queries.cpp
@@ -420,15 +420,15 @@ const char* const DeleteMessageQuery = R"__(
const char* const SetQueueAttributesQuery = R"__(
(
- (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64))))
- (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64))))
- (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64))))
- (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64))))
- (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64))))
+ (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64))))
+ (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64))))
+ (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64))))
+ (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64))))
+ (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64))))
(let contentBasedDeduplication (Parameter 'CONTENT_BASED_DEDUPLICATION (OptionalType (DataType 'Bool))))
- (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64))))
- (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String))))
- (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String))))
+ (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64))))
+ (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String))))
+ (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String))))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
(let attrsTable '%1$s/Attributes)
@@ -441,9 +441,9 @@ const char* const SetQueueAttributesQuery = R"__(
'ReceiveMessageWaitTime
'VisibilityTimeout
'MaximumMessageSize
- 'DlqName
- 'DlqArn
- 'MaxReceiveCount
+ 'DlqName
+ 'DlqArn
+ 'MaxReceiveCount
'ContentBasedDeduplication))
(let attrsRead (SelectRow attrsTable attrsRow attrsSelect))
@@ -453,9 +453,9 @@ const char* const SetQueueAttributesQuery = R"__(
'('ReceiveMessageWaitTime (Coalesce wait (Member attrsRead 'ReceiveMessageWaitTime)))
'('VisibilityTimeout (Coalesce visibility (Member attrsRead 'VisibilityTimeout)))
'('MaximumMessageSize (Coalesce maxMessageSize (Member attrsRead 'MaximumMessageSize)))
- '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount)))
- '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName)))
- '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn)))
+ '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount)))
+ '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName)))
+ '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn)))
'('ContentBasedDeduplication (Coalesce contentBasedDeduplication (Member attrsRead 'ContentBasedDeduplication)))))
(let queuesTable '%5$s/.Queues)
@@ -472,7 +472,7 @@ const char* const SetQueueAttributesQuery = R"__(
'('DlqName (Coalesce dlqName (Member queuesRowRead 'DlqName)))))
(return (AsList
- (UpdateRow attrsTable attrsRow attrsUpdate)
+ (UpdateRow attrsTable attrsRow attrsUpdate)
(UpdateRow queuesTable queuesRow queuesRowUpdate)))
)
)__";
@@ -490,9 +490,9 @@ const char* const InternalGetQueueAttributesQuery = R"__(
'MaximumMessageSize
'MessageRetentionPeriod
'ReceiveMessageWaitTime
- 'MaxReceiveCount
- 'DlqName
- 'DlqArn
+ 'MaxReceiveCount
+ 'DlqName
+ 'DlqArn
'VisibilityTimeout
'ShowDetailedCountersDeadline))
@@ -508,24 +508,24 @@ const char* const ListQueuesQuery = R"__(
(let queuesTable '%5$s/.Queues)
- (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
-
+ (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
+
(let queuesRange '(
'('Account userName userName)
'('QueueName (Utf8String '"") (Void))))
- (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards))
+ (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards))
(let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List))
(let filtered (Filter queues (lambda '(item) (block '(
- (return (Coalesce
- (And
- (Or
- (Equal (Member item 'QueueState) (Uint64 '1))
- (Equal (Member item 'QueueState) (Uint64 '3)))
- (Or
- (Equal (Member item 'FolderId) folderId)
- skipFolderIdFilter))
- (Bool 'false)))
+ (return (Coalesce
+ (And
+ (Or
+ (Equal (Member item 'QueueState) (Uint64 '1))
+ (Equal (Member item 'QueueState) (Uint64 '3)))
+ (Or
+ (Equal (Member item 'FolderId) folderId)
+ skipFolderIdFilter))
+ (Bool 'false)))
)))))
(return (AsList
@@ -613,7 +613,7 @@ const char* const ReadMessageQuery = R"__(
(let dataTable '%1$s/Data)
(let messageTable '%1$s/Messages)
- (let unfilteredResult
+ (let unfilteredResult
(MapParameter keys (lambda '(item) (block '(
(let dataRow '(
'('RandomId (Member item 'RandomId))
@@ -623,8 +623,8 @@ const char* const ReadMessageQuery = R"__(
'DedupId
'Attributes
'Data
- 'MessageId
- 'SenderId))
+ 'MessageId
+ 'SenderId))
(let messageRow '(
'('Offset (Member item 'Offset))))
@@ -635,24 +635,24 @@ const char* const ReadMessageQuery = R"__(
'FirstReceiveTimestamp
'SentTimestamp))
- (let sourceDataFieldsRead (SelectRow dataTable dataRow dataFields))
-
- (return (AsStruct
- '('Valid (Exists sourceDataFieldsRead))
- '('SourceDataFieldsRead sourceDataFieldsRead)
- '('SourceMessageFieldsRead (SelectRow messageTable messageRow messageFields)))))))))
-
- (let result
- (Filter unfilteredResult (lambda '(item) (block '(
- (return (Coalesce (Member item 'Valid) (Bool 'false)))
- )))))
-
+ (let sourceDataFieldsRead (SelectRow dataTable dataRow dataFields))
+
+ (return (AsStruct
+ '('Valid (Exists sourceDataFieldsRead))
+ '('SourceDataFieldsRead sourceDataFieldsRead)
+ '('SourceMessageFieldsRead (SelectRow messageTable messageRow messageFields)))))))))
+
+ (let result
+ (Filter unfilteredResult (lambda '(item) (block '(
+ (return (Coalesce (Member item 'Valid) (Bool 'false)))
+ )))))
+
(return (Extend
(AsList (SetResult 'result result))
- (AsList (SetResult 'movedMessagesCount (Uint64 '0)))
+ (AsList (SetResult 'movedMessagesCount (Uint64 '0)))
(Map result (lambda '(item) (block '(
- (let message (Member item 'SourceMessageFieldsRead))
+ (let message (Member item 'SourceMessageFieldsRead))
(let row '(
'('Offset (Member message 'Offset))))
(let receiveTimestamp
@@ -664,271 +664,271 @@ const char* const ReadMessageQuery = R"__(
)
)__";
-const char* const ReadOrRedriveMessageQuery = R"__(
- (
- (let keys (Parameter 'KEYS
- (ListType (StructType
- '('RandomId (DataType 'Uint64))
- '('GroupId (DataType 'String))
- '('Index (DataType 'Uint64))
- '('Offset (DataType 'Uint64))))))
- (let now (Parameter 'NOW (DataType 'Uint64)))
- (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint32)))
- (let randomId (Parameter 'RANDOM_ID (DataType 'Uint64)))
-
- (let sourceDataTable '%1$s/Data)
- (let sourceStateTable '%1$s/State)
- (let sourceMsgTable '%1$s/Messages)
- (let sourceGroupTable '%1$s/Groups)
- (let sourceSentTsIdx '%1$s/SentTimestampIdx)
-
- (let dlqDataTable '%3$s/Data)
- (let dlqDedupTable '%3$s/Deduplication)
- (let dlqStateTable '%3$s/State)
- (let dlqMsgTable '%3$s/Messages)
- (let dlqGroupTable '%3$s/Groups)
- (let dlqSentTsIdx '%3$s/SentTimestampIdx)
-
- (let unfilteredAllMessages
- (MapParameter keys (lambda '(item) (block '(
- (let dataRow '(
- '('RandomId (Member item 'RandomId))
- '('Offset (Member item 'Offset))))
- (let dataFields '(
- 'RandomId
- 'Offset
- 'DedupId
- 'Attributes
- 'Data
- 'MessageId
- 'SenderId))
-
- (let messageRow '(
- '('Offset (Member item 'Offset))))
- (let messageFields '(
- 'GroupId
- 'Offset
- 'ReceiveCount
- 'FirstReceiveTimestamp
- 'NextOffset
- 'NextRandomId
- 'SentTimestamp))
-
- (let dlqGroupRow '(
- '('GroupId (Member item 'GroupId))))
- (let dlqGroupSelect '(
- 'Head
- 'ReceiveAttemptId
- 'Tail))
- (let dlqGroupRead (SelectRow dlqGroupTable dlqGroupRow dlqGroupSelect))
- (let dlqTail (IfPresent dlqGroupRead (lambda '(x) (Coalesce (Member x 'Tail) (Uint64 '0))) (Uint64 '0)))
-
- (let sourceDataFieldsRead (SelectRow sourceDataTable dataRow dataFields))
- (let sourceMessageFieldsRead (SelectRow sourceMsgTable messageRow messageFields))
-
- (return (AsStruct
- '('Valid (Exists sourceDataFieldsRead))
- '('SourceDataFieldsRead sourceDataFieldsRead)
- '('SourceMessageFieldsRead sourceMessageFieldsRead)
- '('DlqGroupRead dlqGroupRead)
- '('DlqTail dlqTail)
- '('Attributes (Member sourceDataFieldsRead 'Attributes))
- '('Data (Member sourceDataFieldsRead 'Data))
- '('MessageId (Member sourceDataFieldsRead 'MessageId))
- '('SenderId (Member sourceDataFieldsRead 'SenderId))
- '('DeduplicationId (Member sourceDataFieldsRead 'DedupId))
- '('SourceRandomId (Member sourceDataFieldsRead 'RandomId))
- '('SourceOffset (Member sourceMessageFieldsRead 'Offset))
- '('SourceNextOffset (Member sourceMessageFieldsRead 'NextOffset))
- '('SourceNextRandomId (Member sourceMessageFieldsRead 'NextRandomId))
- '('SourceSentTimestamp (Member sourceMessageFieldsRead 'SentTimestamp))
- '('GroupId (Member item 'GroupId))
- '('Delay (Uint64 '0))
- '('Index (Member item 'Index)))))))))
-
- (let allMessages
- (Filter unfilteredAllMessages (lambda '(item) (block '(
- (return (Coalesce (Member item 'Valid) (Bool 'false)))
- )))))
-
- (let messagesToMoveAsStruct
- (Filter allMessages (lambda '(item) (block '(
- (let message (Member item 'SourceMessageFieldsRead))
- (return (Coalesce (GreaterOrEqual (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false)))
- )))))
-
- (let messagesToReturnAsStruct
- (Filter allMessages (lambda '(item) (block '(
- (let message (Member item 'SourceMessageFieldsRead))
- (return (Coalesce (Less (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false)))
- )))))
-
- (let dlqStateRow '(
- '('State (Uint64 '0))))
- (let dlqStateSelect '(
- 'MessageCount
- 'WriteOffset
- 'LastModifiedTimestamp))
- (let dlqStateRead (SelectRow dlqStateTable dlqStateRow dlqStateSelect))
-
- (let dlqSentTimestamp (Max now (Member dlqStateRead 'LastModifiedTimestamp)))
- (let dlqStartOffset (Add (Member dlqStateRead 'WriteOffset) (Uint64 '1)))
- (let dlqNewWriteOffset (Add (Member dlqStateRead 'WriteOffset) (Length messagesToMoveAsStruct)))
-
- (let dlqStateUpdate '(
- '('LastModifiedTimestamp dlqSentTimestamp)
- '('MessageCount (Add (Member dlqStateRead 'MessageCount) (Length messagesToMoveAsStruct)))
- '('WriteOffset dlqNewWriteOffset)))
-
- (let dlqMessagesInfoWithProperIndexes
- (Enumerate messagesToMoveAsStruct (Coalesce dlqStartOffset (Uint64 '1))))
-
- (let dlqMessagesInfoWithProperIndexesSorted
- (Sort dlqMessagesInfoWithProperIndexes (Bool 'true) (lambda '(item) (block '(
- (return (Member (Nth item '1) 'Index))
- ))))
- )
-
- (let sourceStateRow '(
- '('State (Uint64 '0))))
- (let sourceStateSelect '(
- 'MessageCount
- 'LastModifiedTimestamp))
- (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect))
- (let newSourceMsgCount (Sub (Member sourceStateRead 'MessageCount) (Length messagesToMoveAsStruct)))
- (let sourceLastModifiedTimestamp (Max now (Member sourceStateRead 'LastModifiedTimestamp)))
-
- (let sourceStateUpdate '(
- '('LastModifiedTimestamp sourceLastModifiedTimestamp)
- '('MessageCount newSourceMsgCount)))
-
- (return (Extend
- (AsList (SetResult 'result messagesToReturnAsStruct))
- (AsList (SetResult 'movedMessagesCount (Length messagesToMoveAsStruct)))
+const char* const ReadOrRedriveMessageQuery = R"__(
+ (
+ (let keys (Parameter 'KEYS
+ (ListType (StructType
+ '('RandomId (DataType 'Uint64))
+ '('GroupId (DataType 'String))
+ '('Index (DataType 'Uint64))
+ '('Offset (DataType 'Uint64))))))
+ (let now (Parameter 'NOW (DataType 'Uint64)))
+ (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (DataType 'Uint32)))
+ (let randomId (Parameter 'RANDOM_ID (DataType 'Uint64)))
+
+ (let sourceDataTable '%1$s/Data)
+ (let sourceStateTable '%1$s/State)
+ (let sourceMsgTable '%1$s/Messages)
+ (let sourceGroupTable '%1$s/Groups)
+ (let sourceSentTsIdx '%1$s/SentTimestampIdx)
+
+ (let dlqDataTable '%3$s/Data)
+ (let dlqDedupTable '%3$s/Deduplication)
+ (let dlqStateTable '%3$s/State)
+ (let dlqMsgTable '%3$s/Messages)
+ (let dlqGroupTable '%3$s/Groups)
+ (let dlqSentTsIdx '%3$s/SentTimestampIdx)
+
+ (let unfilteredAllMessages
+ (MapParameter keys (lambda '(item) (block '(
+ (let dataRow '(
+ '('RandomId (Member item 'RandomId))
+ '('Offset (Member item 'Offset))))
+ (let dataFields '(
+ 'RandomId
+ 'Offset
+ 'DedupId
+ 'Attributes
+ 'Data
+ 'MessageId
+ 'SenderId))
+
+ (let messageRow '(
+ '('Offset (Member item 'Offset))))
+ (let messageFields '(
+ 'GroupId
+ 'Offset
+ 'ReceiveCount
+ 'FirstReceiveTimestamp
+ 'NextOffset
+ 'NextRandomId
+ 'SentTimestamp))
+
+ (let dlqGroupRow '(
+ '('GroupId (Member item 'GroupId))))
+ (let dlqGroupSelect '(
+ 'Head
+ 'ReceiveAttemptId
+ 'Tail))
+ (let dlqGroupRead (SelectRow dlqGroupTable dlqGroupRow dlqGroupSelect))
+ (let dlqTail (IfPresent dlqGroupRead (lambda '(x) (Coalesce (Member x 'Tail) (Uint64 '0))) (Uint64 '0)))
+
+ (let sourceDataFieldsRead (SelectRow sourceDataTable dataRow dataFields))
+ (let sourceMessageFieldsRead (SelectRow sourceMsgTable messageRow messageFields))
+
+ (return (AsStruct
+ '('Valid (Exists sourceDataFieldsRead))
+ '('SourceDataFieldsRead sourceDataFieldsRead)
+ '('SourceMessageFieldsRead sourceMessageFieldsRead)
+ '('DlqGroupRead dlqGroupRead)
+ '('DlqTail dlqTail)
+ '('Attributes (Member sourceDataFieldsRead 'Attributes))
+ '('Data (Member sourceDataFieldsRead 'Data))
+ '('MessageId (Member sourceDataFieldsRead 'MessageId))
+ '('SenderId (Member sourceDataFieldsRead 'SenderId))
+ '('DeduplicationId (Member sourceDataFieldsRead 'DedupId))
+ '('SourceRandomId (Member sourceDataFieldsRead 'RandomId))
+ '('SourceOffset (Member sourceMessageFieldsRead 'Offset))
+ '('SourceNextOffset (Member sourceMessageFieldsRead 'NextOffset))
+ '('SourceNextRandomId (Member sourceMessageFieldsRead 'NextRandomId))
+ '('SourceSentTimestamp (Member sourceMessageFieldsRead 'SentTimestamp))
+ '('GroupId (Member item 'GroupId))
+ '('Delay (Uint64 '0))
+ '('Index (Member item 'Index)))))))))
+
+ (let allMessages
+ (Filter unfilteredAllMessages (lambda '(item) (block '(
+ (return (Coalesce (Member item 'Valid) (Bool 'false)))
+ )))))
+
+ (let messagesToMoveAsStruct
+ (Filter allMessages (lambda '(item) (block '(
+ (let message (Member item 'SourceMessageFieldsRead))
+ (return (Coalesce (GreaterOrEqual (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false)))
+ )))))
+
+ (let messagesToReturnAsStruct
+ (Filter allMessages (lambda '(item) (block '(
+ (let message (Member item 'SourceMessageFieldsRead))
+ (return (Coalesce (Less (Member message 'ReceiveCount) maxReceiveCount) (Bool 'false)))
+ )))))
+
+ (let dlqStateRow '(
+ '('State (Uint64 '0))))
+ (let dlqStateSelect '(
+ 'MessageCount
+ 'WriteOffset
+ 'LastModifiedTimestamp))
+ (let dlqStateRead (SelectRow dlqStateTable dlqStateRow dlqStateSelect))
+
+ (let dlqSentTimestamp (Max now (Member dlqStateRead 'LastModifiedTimestamp)))
+ (let dlqStartOffset (Add (Member dlqStateRead 'WriteOffset) (Uint64 '1)))
+ (let dlqNewWriteOffset (Add (Member dlqStateRead 'WriteOffset) (Length messagesToMoveAsStruct)))
+
+ (let dlqStateUpdate '(
+ '('LastModifiedTimestamp dlqSentTimestamp)
+ '('MessageCount (Add (Member dlqStateRead 'MessageCount) (Length messagesToMoveAsStruct)))
+ '('WriteOffset dlqNewWriteOffset)))
+
+ (let dlqMessagesInfoWithProperIndexes
+ (Enumerate messagesToMoveAsStruct (Coalesce dlqStartOffset (Uint64 '1))))
+
+ (let dlqMessagesInfoWithProperIndexesSorted
+ (Sort dlqMessagesInfoWithProperIndexes (Bool 'true) (lambda '(item) (block '(
+ (return (Member (Nth item '1) 'Index))
+ ))))
+ )
+
+ (let sourceStateRow '(
+ '('State (Uint64 '0))))
+ (let sourceStateSelect '(
+ 'MessageCount
+ 'LastModifiedTimestamp))
+ (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect))
+ (let newSourceMsgCount (Sub (Member sourceStateRead 'MessageCount) (Length messagesToMoveAsStruct)))
+ (let sourceLastModifiedTimestamp (Max now (Member sourceStateRead 'LastModifiedTimestamp)))
+
+ (let sourceStateUpdate '(
+ '('LastModifiedTimestamp sourceLastModifiedTimestamp)
+ '('MessageCount newSourceMsgCount)))
+
+ (return (Extend
+ (AsList (SetResult 'result messagesToReturnAsStruct))
+ (AsList (SetResult 'movedMessagesCount (Length messagesToMoveAsStruct)))
(AsList (SetResult 'newMessagesCount newSourceMsgCount))
- (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow dlqStateTable dlqStateRow dlqStateUpdate))
- (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate))
-
- # copy messages to dlq
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let dataRow '(
- '('RandomId randomId)
- '('Offset (Nth item '0))))
- (let dataUpdate '(
- '('Data (Member (Nth item '1) 'Data))
- '('DedupId (Member (Nth item '1) 'DeduplicationId))
- '('Attributes (Member (Nth item '1) 'Attributes))
- '('SenderId (Member (Nth item '1) 'SenderId))
- '('MessageId (Member (Nth item '1) 'MessageId))))
- (return (UpdateRow dlqDataTable dataRow dataUpdate))))))
-
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let msgRow '(
- '('Offset (Nth item '0))))
- (let messageUpdate '(
- '('RandomId randomId)
- '('GroupId (Member (Nth item '1) 'GroupId))
- '('NextOffset (Uint64 '0))
- '('NextRandomId (Uint64 '0))
- '('ReceiveCount (Uint32 '0))
- '('FirstReceiveTimestamp (Uint64 '0))
- '('SentTimestamp dlqSentTimestamp)))
- (return (UpdateRow dlqMsgTable msgRow messageUpdate))))))
-
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let sentTsRow '(
- '('SentTimestamp dlqSentTimestamp)
- '('Offset (Nth item '0))))
- (let delay (Member (Nth item '1) 'Delay))
- (let delayDeadline (Uint64 '0))
- (let sentTsUpdate '(
- '('RandomId randomId)
- '('DelayDeadline delayDeadline)
- '('GroupId (Member (Nth item '1) 'GroupId))))
- (return (UpdateRow dlqSentTsIdx sentTsRow sentTsUpdate))))))
-
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let groupRow '(
- '('GroupId (Member (Nth item '1) 'GroupId))))
- (let delay (Member (Nth item '1) 'Delay))
- (let delayDeadline (Uint64 '0))
- (let groupInsert '(
- '('RandomId randomId)
- '('Head (Nth item '0))
- '('Tail (Nth item '0))
- '('LockTimestamp (Uint64 '0))
- '('VisibilityDeadline (Uint64 '0))))
- (let dlqGroupRead (Member (Nth item '1) 'DlqGroupRead))
- (let groupUpdate '(
- '('Head (Member dlqGroupRead 'Head))
- '('Tail (Nth item '0))))
- (let dlqTail (Member (Nth item '1) 'DlqTail))
- (return
- (If (Equal dlqTail (Uint64 '0))
- (UpdateRow dlqGroupTable groupRow groupInsert)
- (UpdateRow dlqGroupTable groupRow groupUpdate)
- )
- )))))
-
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let dlqTail (Member (Nth item '1) 'DlqTail))
- (let prevMessageRow '(
- '('Offset dlqTail)))
- (let prevMessageUpdate '(
- '('NextOffset (Nth item '0))
- '('NextRandomId randomId)))
- (return
- (If (NotEqual dlqTail (Uint64 '0))
- (UpdateRow dlqMsgTable prevMessageRow prevMessageUpdate)
- (Void))
- )))))
-
- # remove dead letters' content from source queue
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let row '(
- '('RandomId (Member (Nth item '1) 'SourceRandomId))
- '('Offset (Member (Nth item '1) 'SourceOffset))))
- (return (EraseRow sourceDataTable row))))))
-
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let row '(
- '('Offset (Member (Nth item '1) 'SourceOffset))))
- (return (EraseRow sourceMsgTable row))))))
-
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let row '(
- '('SentTimestamp (Member (Nth item '1) 'SourceSentTimestamp))
- '('Offset (Member (Nth item '1) 'SourceOffset))))
- (return (EraseRow sourceSentTsIdx row))))))
-
- (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
- (let row '(
- '('GroupId (Member (Nth item '1) 'GroupId))))
- (let update '(
- '('RandomId (Member (Nth item '1) 'SourceNextRandomId))
- '('Head (Member (Nth item '1) 'SourceNextOffset))
- '('LockTimestamp (Uint64 '0))
- '('VisibilityDeadline (Uint64 '0))))
-
- (return
- (If (Coalesce (Equal (Member (Nth item '1) 'SourceNextOffset) (Uint64 '0)) (Bool 'false))
- (EraseRow sourceGroupTable row)
- (UpdateRow sourceGroupTable row update)))))))
-
- # just return ordinary messages
- (Map messagesToReturnAsStruct (lambda '(item) (block '(
- (let message (Member item 'SourceMessageFieldsRead))
- (let row '(
- '('Offset (Member message 'Offset))))
- (let receiveTimestamp
- (If (Coalesce (Equal (Member message 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member message 'FirstReceiveTimestamp)))
- (let update '(
- '('FirstReceiveTimestamp receiveTimestamp)
- '('ReceiveCount (Add (Member message 'ReceiveCount) (Uint32 '1)))))
- (return (UpdateRow sourceMsgTable row update))))))))
- )
-)__";
-
+ (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow dlqStateTable dlqStateRow dlqStateUpdate))
+ (ListIf (HasItems messagesToMoveAsStruct) (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate))
+
+ # copy messages to dlq
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let dataRow '(
+ '('RandomId randomId)
+ '('Offset (Nth item '0))))
+ (let dataUpdate '(
+ '('Data (Member (Nth item '1) 'Data))
+ '('DedupId (Member (Nth item '1) 'DeduplicationId))
+ '('Attributes (Member (Nth item '1) 'Attributes))
+ '('SenderId (Member (Nth item '1) 'SenderId))
+ '('MessageId (Member (Nth item '1) 'MessageId))))
+ (return (UpdateRow dlqDataTable dataRow dataUpdate))))))
+
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let msgRow '(
+ '('Offset (Nth item '0))))
+ (let messageUpdate '(
+ '('RandomId randomId)
+ '('GroupId (Member (Nth item '1) 'GroupId))
+ '('NextOffset (Uint64 '0))
+ '('NextRandomId (Uint64 '0))
+ '('ReceiveCount (Uint32 '0))
+ '('FirstReceiveTimestamp (Uint64 '0))
+ '('SentTimestamp dlqSentTimestamp)))
+ (return (UpdateRow dlqMsgTable msgRow messageUpdate))))))
+
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let sentTsRow '(
+ '('SentTimestamp dlqSentTimestamp)
+ '('Offset (Nth item '0))))
+ (let delay (Member (Nth item '1) 'Delay))
+ (let delayDeadline (Uint64 '0))
+ (let sentTsUpdate '(
+ '('RandomId randomId)
+ '('DelayDeadline delayDeadline)
+ '('GroupId (Member (Nth item '1) 'GroupId))))
+ (return (UpdateRow dlqSentTsIdx sentTsRow sentTsUpdate))))))
+
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let groupRow '(
+ '('GroupId (Member (Nth item '1) 'GroupId))))
+ (let delay (Member (Nth item '1) 'Delay))
+ (let delayDeadline (Uint64 '0))
+ (let groupInsert '(
+ '('RandomId randomId)
+ '('Head (Nth item '0))
+ '('Tail (Nth item '0))
+ '('LockTimestamp (Uint64 '0))
+ '('VisibilityDeadline (Uint64 '0))))
+ (let dlqGroupRead (Member (Nth item '1) 'DlqGroupRead))
+ (let groupUpdate '(
+ '('Head (Member dlqGroupRead 'Head))
+ '('Tail (Nth item '0))))
+ (let dlqTail (Member (Nth item '1) 'DlqTail))
+ (return
+ (If (Equal dlqTail (Uint64 '0))
+ (UpdateRow dlqGroupTable groupRow groupInsert)
+ (UpdateRow dlqGroupTable groupRow groupUpdate)
+ )
+ )))))
+
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let dlqTail (Member (Nth item '1) 'DlqTail))
+ (let prevMessageRow '(
+ '('Offset dlqTail)))
+ (let prevMessageUpdate '(
+ '('NextOffset (Nth item '0))
+ '('NextRandomId randomId)))
+ (return
+ (If (NotEqual dlqTail (Uint64 '0))
+ (UpdateRow dlqMsgTable prevMessageRow prevMessageUpdate)
+ (Void))
+ )))))
+
+ # remove dead letters' content from source queue
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let row '(
+ '('RandomId (Member (Nth item '1) 'SourceRandomId))
+ '('Offset (Member (Nth item '1) 'SourceOffset))))
+ (return (EraseRow sourceDataTable row))))))
+
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let row '(
+ '('Offset (Member (Nth item '1) 'SourceOffset))))
+ (return (EraseRow sourceMsgTable row))))))
+
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let row '(
+ '('SentTimestamp (Member (Nth item '1) 'SourceSentTimestamp))
+ '('Offset (Member (Nth item '1) 'SourceOffset))))
+ (return (EraseRow sourceSentTsIdx row))))))
+
+ (Map dlqMessagesInfoWithProperIndexesSorted (lambda '(item) (block '(
+ (let row '(
+ '('GroupId (Member (Nth item '1) 'GroupId))))
+ (let update '(
+ '('RandomId (Member (Nth item '1) 'SourceNextRandomId))
+ '('Head (Member (Nth item '1) 'SourceNextOffset))
+ '('LockTimestamp (Uint64 '0))
+ '('VisibilityDeadline (Uint64 '0))))
+
+ (return
+ (If (Coalesce (Equal (Member (Nth item '1) 'SourceNextOffset) (Uint64 '0)) (Bool 'false))
+ (EraseRow sourceGroupTable row)
+ (UpdateRow sourceGroupTable row update)))))))
+
+ # just return ordinary messages
+ (Map messagesToReturnAsStruct (lambda '(item) (block '(
+ (let message (Member item 'SourceMessageFieldsRead))
+ (let row '(
+ '('Offset (Member message 'Offset))))
+ (let receiveTimestamp
+ (If (Coalesce (Equal (Member message 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member message 'FirstReceiveTimestamp)))
+ (let update '(
+ '('FirstReceiveTimestamp receiveTimestamp)
+ '('ReceiveCount (Add (Member message 'ReceiveCount) (Uint32 '1)))))
+ (return (UpdateRow sourceMsgTable row update))))))))
+ )
+)__";
+
const char* const WriteMessageQuery = R"__(
(
(let randomId (Parameter 'RANDOM_ID (DataType 'Uint64)))
@@ -1333,53 +1333,53 @@ const char* const GetRetentionOffsetQuery = R"__(
)
)__";
-const char* const ListDeadLetterSourceQueuesQuery = R"__(
- (
+const char* const ListDeadLetterSourceQueuesQuery = R"__(
+ (
(let folderId (Parameter 'FOLDERID (DataType 'Utf8String)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
-
+
(let queuesTable '%5$s/.Queues)
(let queuesRow '(
'('Account userName)
- '('QueueName (Utf8String '"%4$s"))))
-
+ '('QueueName (Utf8String '"%4$s"))))
+
(let queuesRowSelect '(
- 'QueueName
- 'CustomQueueName))
-
+ 'QueueName
+ 'CustomQueueName))
+
(let queuesRowRead (SelectRow queuesTable queuesRow queuesRowSelect))
-
- (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
-
- (let dlqName
+
+ (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
+
+ (let dlqName
(If skipFolderIdFilter (Member queuesRowRead 'QueueName) (Coalesce (Member queuesRowRead 'CustomQueueName) (Utf8String '""))))
-
+
(let queuesRange '(
'('Account userName userName)
- '('QueueName (Utf8String '"") (Void))))
+ '('QueueName (Utf8String '"") (Void))))
(let queuesSelect '('QueueName 'QueueState 'FolderId 'DlqName 'CustomQueueName))
(let queues (Member (SelectRange queuesTable queuesRange queuesSelect '()) 'List))
-
- (let filtered (Filter queues (lambda '(item) (block '(
- (return (Coalesce
- (And
- (And
- (Or
- (Equal (Member item 'QueueState) (Uint64 '1))
- (Equal (Member item 'QueueState) (Uint64 '3)))
- (Or
- (Equal (Member item 'FolderId) folderId)
- skipFolderIdFilter))
- (Equal (Member item 'DlqName) dlqName))
- (Bool 'false)))
- )))))
-
- (return (AsList
- (SetResult 'queues filtered)))
- )
-)__";
-
+
+ (let filtered (Filter queues (lambda '(item) (block '(
+ (return (Coalesce
+ (And
+ (And
+ (Or
+ (Equal (Member item 'QueueState) (Uint64 '1))
+ (Equal (Member item 'QueueState) (Uint64 '3)))
+ (Or
+ (Equal (Member item 'FolderId) folderId)
+ skipFolderIdFilter))
+ (Equal (Member item 'DlqName) dlqName))
+ (Bool 'false)))
+ )))))
+
+ (return (AsList
+ (SetResult 'queues filtered)))
+ )
+)__";
+
} // namespace
const char* GetFifoQueryById(size_t id) {
@@ -1416,10 +1416,10 @@ const char* GetFifoQueryById(size_t id) {
return GetOldestMessageTimestampMetricsQuery;
case GET_RETENTION_OFFSET_ID: // 17
return GetRetentionOffsetQuery;
- case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 18
- return ListDeadLetterSourceQueuesQuery;
- case READ_OR_REDRIVE_MESSAGE_ID: // 22
- return ReadOrRedriveMessageQuery;
+ case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 18
+ return ListDeadLetterSourceQueuesQuery;
+ case READ_OR_REDRIVE_MESSAGE_ID: // 22
+ return ReadOrRedriveMessageQuery;
}
return nullptr;
diff --git a/ydb/core/ymq/queues/fifo/schema.cpp b/ydb/core/ymq/queues/fifo/schema.cpp
index da394c7599..49e04d773b 100644
--- a/ydb/core/ymq/queues/fifo/schema.cpp
+++ b/ydb/core/ymq/queues/fifo/schema.cpp
@@ -10,9 +10,9 @@ static const TVector<TColumn> AttributesColumns = {
TColumn("MaximumMessageSize", NScheme::NTypeIds::Uint64),
TColumn("MessageRetentionPeriod", NScheme::NTypeIds::Uint64),
TColumn("ReceiveMessageWaitTime", NScheme::NTypeIds::Uint64),
- TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64),
- TColumn("DlqName", NScheme::NTypeIds::Utf8),
- TColumn("DlqArn", NScheme::NTypeIds::Utf8),
+ TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64),
+ TColumn("DlqName", NScheme::NTypeIds::Utf8),
+ TColumn("DlqArn", NScheme::NTypeIds::Utf8),
TColumn("MaxReceiveCount", NScheme::NTypeIds::Uint64),
TColumn("ShowDetailedCountersDeadline", NScheme::NTypeIds::Uint64)};
@@ -36,7 +36,7 @@ static const TVector<TColumn> DataColumns = {
TColumn("Attributes", NScheme::NTypeIds::String),
TColumn("Data", NScheme::NTypeIds::String),
TColumn("MessageId", NScheme::NTypeIds::String),
- TColumn("SenderId", NScheme::NTypeIds::String),
+ TColumn("SenderId", NScheme::NTypeIds::String),
};
static const TVector<TColumn> MessagesColumns = {
diff --git a/ydb/core/ymq/queues/std/queries.cpp b/ydb/core/ymq/queues/std/queries.cpp
index 442ae6389f..202d9c1489 100644
--- a/ydb/core/ymq/queues/std/queries.cpp
+++ b/ydb/core/ymq/queues/std/queries.cpp
@@ -415,14 +415,14 @@ const char* const DeleteMessageQuery = R"__(
const char* const SetQueueAttributesQuery = R"__(
(
- (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64))))
- (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64))))
- (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64))))
- (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64))))
- (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64))))
- (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64))))
- (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String))))
- (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String))))
+ (let delay (Parameter 'DELAY (OptionalType (DataType 'Uint64))))
+ (let retention (Parameter 'RETENTION (OptionalType (DataType 'Uint64))))
+ (let visibility (Parameter 'VISIBILITY (OptionalType (DataType 'Uint64))))
+ (let wait (Parameter 'WAIT (OptionalType (DataType 'Uint64))))
+ (let maxMessageSize (Parameter 'MAX_MESSAGE_SIZE (OptionalType (DataType 'Uint64))))
+ (let maxReceiveCount (Parameter 'MAX_RECEIVE_COUNT (OptionalType (DataType 'Uint64))))
+ (let dlqArn (Parameter 'DLQ_TARGET_ARN (OptionalType (DataType 'Utf8String))))
+ (let dlqName (Parameter 'DLQ_TARGET_NAME (OptionalType (DataType 'Utf8String))))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
(let attrsTable '%1$s/Attributes)
@@ -434,10 +434,10 @@ const char* const SetQueueAttributesQuery = R"__(
'MessageRetentionPeriod
'ReceiveMessageWaitTime
'VisibilityTimeout
- 'MaximumMessageSize
- 'DlqName
- 'DlqArn
- 'MaxReceiveCount))
+ 'MaximumMessageSize
+ 'DlqName
+ 'DlqArn
+ 'MaxReceiveCount))
(let attrsRead (SelectRow attrsTable attrsRow attrsSelect))
(let attrsUpdate '(
@@ -445,9 +445,9 @@ const char* const SetQueueAttributesQuery = R"__(
'('MessageRetentionPeriod (Coalesce retention (Member attrsRead 'MessageRetentionPeriod)))
'('ReceiveMessageWaitTime (Coalesce wait (Member attrsRead 'ReceiveMessageWaitTime)))
'('VisibilityTimeout (Coalesce visibility (Member attrsRead 'VisibilityTimeout)))
- '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount)))
- '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName)))
- '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn)))
+ '('MaxReceiveCount (Coalesce maxReceiveCount (Member attrsRead 'MaxReceiveCount)))
+ '('DlqName (Coalesce dlqName (Member attrsRead 'DlqName)))
+ '('DlqArn (Coalesce dlqArn (Member attrsRead 'DlqArn)))
'('MaximumMessageSize (Coalesce maxMessageSize (Member attrsRead 'MaximumMessageSize)))))
(let queuesTable '%5$s/.Queues)
@@ -464,7 +464,7 @@ const char* const SetQueueAttributesQuery = R"__(
'('DlqName (Coalesce dlqName (Member queuesRowRead 'DlqName)))))
(return (AsList
- (UpdateRow attrsTable attrsRow attrsUpdate)
+ (UpdateRow attrsTable attrsRow attrsUpdate)
(UpdateRow queuesTable queuesRow queuesRowUpdate)))
)
)__";
@@ -482,9 +482,9 @@ const char* const InternalGetQueueAttributesQuery = R"__(
'MaximumMessageSize
'MessageRetentionPeriod
'ReceiveMessageWaitTime
- 'MaxReceiveCount
- 'DlqName
- 'DlqArn
+ 'MaxReceiveCount
+ 'DlqName
+ 'DlqArn
'VisibilityTimeout
'ShowDetailedCountersDeadline))
@@ -500,24 +500,24 @@ const char* const ListQueuesQuery = R"__(
(let queuesTable '%5$s/.Queues)
- (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
-
+ (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
+
(let queuesRange '(
'('Account userName userName)
'('QueueName (Utf8String '"") (Void))))
- (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards))
+ (let queueSelect '('QueueName 'QueueId 'QueueState 'FifoQueue 'CreatedTimestamp 'CustomQueueName 'FolderId 'MasterTabletId 'Version 'Shards))
(let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List))
(let filtered (Filter queues (lambda '(item) (block '(
- (return (Coalesce
- (And
- (Or
- (Equal (Member item 'QueueState) (Uint64 '1))
- (Equal (Member item 'QueueState) (Uint64 '3)))
- (Or
- (Equal (Member item 'FolderId) folderId)
- skipFolderIdFilter))
- (Bool 'false)))
+ (return (Coalesce
+ (And
+ (Or
+ (Equal (Member item 'QueueState) (Uint64 '1))
+ (Equal (Member item 'QueueState) (Uint64 '3)))
+ (Or
+ (Equal (Member item 'FolderId) folderId)
+ skipFolderIdFilter))
+ (Bool 'false)))
)))))
(return (AsList
@@ -532,8 +532,8 @@ const char* const LoadMessageQuery = R"__(
'('RandomId (DataType 'Uint64))
'('Offset (DataType 'Uint64))
'('CurrentVisibilityDeadline (DataType 'Uint64))
- '('DlqIndex (DataType 'Uint64))
- '('IsDeadLetter (DataType 'Bool))
+ '('DlqIndex (DataType 'Uint64))
+ '('IsDeadLetter (DataType 'Bool))
'('VisibilityDeadline (DataType 'Uint64))))))
(let now (Parameter 'NOW (DataType 'Uint64)))
(let readId (Parameter 'READ_ID (DataType 'Uint64)))
@@ -563,7 +563,7 @@ const char* const LoadMessageQuery = R"__(
(let fields '(
'Attributes
'Data
- 'SenderId
+ 'SenderId
'MessageId))
(return (SelectRow dataTable row fields)))))
@@ -596,12 +596,12 @@ const char* const LoadMessageQuery = R"__(
'('LoadId readId)
'('Attributes (Member data 'Attributes))
'('Data (Member data 'Data))
- '('SenderId (Member data 'SenderId))
+ '('SenderId (Member data 'SenderId))
'('MessageId (Member data 'MessageId))
'('FirstReceiveTimestamp receiveTimestamp)
'('LockTimestamp now)
- '('IsDeadLetter (Member item 'IsDeadLetter))
- '('DlqIndex (Member item 'DlqIndex))
+ '('IsDeadLetter (Member item 'IsDeadLetter))
+ '('DlqIndex (Member item 'DlqIndex))
'('ReceiveCount (Add (Member read 'ReceiveCount) (Uint32 '1)))
'('SentTimestamp (Member read 'SentTimestamp))
'('VisibilityDeadline (If valid (Member item 'VisibilityDeadline) (Member read 'VisibilityDeadline)))
@@ -615,7 +615,7 @@ const char* const LoadMessageQuery = R"__(
(return (Extend
(AsList (SetResult 'result records))
- (AsList (SetResult 'movedMessagesCount (Uint64 '0)))
+ (AsList (SetResult 'movedMessagesCount (Uint64 '0)))
(Map result (lambda '(item) (block '(
(let row '(
@@ -630,204 +630,204 @@ const char* const LoadMessageQuery = R"__(
)
)__";
-const char* const LoadOrRedriveMessageQuery = R"__(
- (
- (let keys (Parameter 'KEYS
- (ListType (StructType
- '('RandomId (DataType 'Uint64))
- '('Offset (DataType 'Uint64))
- '('CurrentVisibilityDeadline (DataType 'Uint64))
- '('DlqIndex (DataType 'Uint64))
- '('IsDeadLetter (DataType 'Bool))
- '('VisibilityDeadline (DataType 'Uint64))))))
-
- (let now (Parameter 'NOW (DataType 'Uint64)))
- (let readId (Parameter 'READ_ID (DataType 'Uint64)))
- (let shard (Parameter 'SHARD (DataType 'Uint64)))
- (let deadLettersCount (Parameter 'DEAD_LETTERS_COUNT (DataType 'Uint64)))
-
- (let sourceMessageDataTable '%1$s/%2$lu/MessageData)
- (let sourceSentTsIdxTable '%1$s/%2$lu/SentTimestampIdx)
- (let sourceInflyTable '%1$s/%2$lu/Infly)
- (let sourceStateTable '%1$s/State)
-
- (let deadLetterMessageDataTable '%3$s/%4$lu/MessageData)
- (let deadLetterMessagesTable '%3$s/%4$lu/Messages)
- (let deadLetterSentTsIdxTable '%3$s/%4$lu/SentTimestampIdx)
- (let deadLetterStateTable '%3$s/State)
-
- (let sourceStateRow '(
- '('State (Uint64 '%2$lu))))
- (let sourceStateSelect '(
- 'MessageCount
- 'LastModifiedTimestamp
- 'InflyCount))
- (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect))
-
- (let records
- (MapParameter keys (lambda '(item) (block '(
- (let read (block '(
- (let row '(
- '('Offset (Member item 'Offset))))
- (let fields '(
- 'LoadId
- 'FirstReceiveTimestamp
- 'ReceiveCount
- 'SentTimestamp
- 'VisibilityDeadline))
- (return (SelectRow sourceInflyTable row fields)))))
-
- (let data (block '(
- (let row '(
- '('RandomId (Member item 'RandomId))
- '('Offset (Member item 'Offset))))
- (let fields '(
- 'Attributes
- 'Data
- 'SenderId
- 'MessageId))
- (return (SelectRow sourceMessageDataTable row fields)))))
-
- (let receiveTimestamp
- (If (Coalesce (Equal (Member read 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member read 'FirstReceiveTimestamp)))
-
- (let valid
- (Coalesce
- (Or
- (Equal (Member read 'VisibilityDeadline) (Member item 'CurrentVisibilityDeadline))
- (And
- (Equal (Member read 'LoadId) readId)
- (Equal (Member read 'VisibilityDeadline) (Member item 'VisibilityDeadline))))
- (Bool 'false)))
-
- (return
- (AsStruct
- '('Offset (Member item 'Offset))
- '('RandomId (Member item 'RandomId))
- '('IsDeadLetter (Member item 'IsDeadLetter))
- '('DlqIndex (Member item 'DlqIndex))
- '('LoadId readId)
- '('Attributes (Member data 'Attributes))
- '('Data (Member data 'Data))
- '('SenderId (Member data 'SenderId))
- '('MessageId (Member data 'MessageId))
- '('FirstReceiveTimestamp receiveTimestamp)
- '('LockTimestamp now)
- '('ReceiveCount (Add (Member read 'ReceiveCount) (Uint32 '1)))
- '('SentTimestamp (Member read 'SentTimestamp))
- '('VisibilityDeadline (If valid (Member item 'VisibilityDeadline) (Member read 'VisibilityDeadline)))
- '('Valid valid)
- '('Exists (Exists read))))
- )))))
-
- (let validMessages
- (Filter records (lambda '(item) (block '(
- (return (Coalesce (Member item 'Valid) (Bool 'false))))))))
-
- (let messagesToMove
- (Filter validMessages (lambda '(item) (block '(
- (return (Coalesce (Member item 'IsDeadLetter) (Bool 'false))))))))
-
- (let messagesToUpdate
- (Filter validMessages (lambda '(item) (block '(
- (return (Coalesce (Not (Member item 'IsDeadLetter)) (Bool 'false))))))))
-
- (let deadLetterStateRow '(
- '('State (Uint64 '%4$lu))))
- (let deadLetterStateSelect '(
- 'MessageCount
- 'WriteOffset
- 'LastModifiedTimestamp))
- (let deadLetterStateRead (SelectRow deadLetterStateTable deadLetterStateRow deadLetterStateSelect))
-
- (let newDlqMessagesCount (Add (Member deadLetterStateRead 'MessageCount) (Length messagesToMove)))
- (let newDlqWriteOffset (Add (Member deadLetterStateRead 'WriteOffset) deadLettersCount))
- (let dlqStartOffset (Add (Member deadLetterStateRead 'WriteOffset) (Uint64 '1)))
-
- (let dlqMostRecentTimestamp (Max (Member deadLetterStateRead 'LastModifiedTimestamp) now))
-
- (let deadLetterStateUpdate '(
- '('LastModifiedTimestamp dlqMostRecentTimestamp)
- '('MessageCount newDlqMessagesCount)
- '('WriteOffset newDlqWriteOffset)))
-
+const char* const LoadOrRedriveMessageQuery = R"__(
+ (
+ (let keys (Parameter 'KEYS
+ (ListType (StructType
+ '('RandomId (DataType 'Uint64))
+ '('Offset (DataType 'Uint64))
+ '('CurrentVisibilityDeadline (DataType 'Uint64))
+ '('DlqIndex (DataType 'Uint64))
+ '('IsDeadLetter (DataType 'Bool))
+ '('VisibilityDeadline (DataType 'Uint64))))))
+
+ (let now (Parameter 'NOW (DataType 'Uint64)))
+ (let readId (Parameter 'READ_ID (DataType 'Uint64)))
+ (let shard (Parameter 'SHARD (DataType 'Uint64)))
+ (let deadLettersCount (Parameter 'DEAD_LETTERS_COUNT (DataType 'Uint64)))
+
+ (let sourceMessageDataTable '%1$s/%2$lu/MessageData)
+ (let sourceSentTsIdxTable '%1$s/%2$lu/SentTimestampIdx)
+ (let sourceInflyTable '%1$s/%2$lu/Infly)
+ (let sourceStateTable '%1$s/State)
+
+ (let deadLetterMessageDataTable '%3$s/%4$lu/MessageData)
+ (let deadLetterMessagesTable '%3$s/%4$lu/Messages)
+ (let deadLetterSentTsIdxTable '%3$s/%4$lu/SentTimestampIdx)
+ (let deadLetterStateTable '%3$s/State)
+
+ (let sourceStateRow '(
+ '('State (Uint64 '%2$lu))))
+ (let sourceStateSelect '(
+ 'MessageCount
+ 'LastModifiedTimestamp
+ 'InflyCount))
+ (let sourceStateRead (SelectRow sourceStateTable sourceStateRow sourceStateSelect))
+
+ (let records
+ (MapParameter keys (lambda '(item) (block '(
+ (let read (block '(
+ (let row '(
+ '('Offset (Member item 'Offset))))
+ (let fields '(
+ 'LoadId
+ 'FirstReceiveTimestamp
+ 'ReceiveCount
+ 'SentTimestamp
+ 'VisibilityDeadline))
+ (return (SelectRow sourceInflyTable row fields)))))
+
+ (let data (block '(
+ (let row '(
+ '('RandomId (Member item 'RandomId))
+ '('Offset (Member item 'Offset))))
+ (let fields '(
+ 'Attributes
+ 'Data
+ 'SenderId
+ 'MessageId))
+ (return (SelectRow sourceMessageDataTable row fields)))))
+
+ (let receiveTimestamp
+ (If (Coalesce (Equal (Member read 'FirstReceiveTimestamp) (Uint64 '0)) (Bool 'false)) now (Member read 'FirstReceiveTimestamp)))
+
+ (let valid
+ (Coalesce
+ (Or
+ (Equal (Member read 'VisibilityDeadline) (Member item 'CurrentVisibilityDeadline))
+ (And
+ (Equal (Member read 'LoadId) readId)
+ (Equal (Member read 'VisibilityDeadline) (Member item 'VisibilityDeadline))))
+ (Bool 'false)))
+
+ (return
+ (AsStruct
+ '('Offset (Member item 'Offset))
+ '('RandomId (Member item 'RandomId))
+ '('IsDeadLetter (Member item 'IsDeadLetter))
+ '('DlqIndex (Member item 'DlqIndex))
+ '('LoadId readId)
+ '('Attributes (Member data 'Attributes))
+ '('Data (Member data 'Data))
+ '('SenderId (Member data 'SenderId))
+ '('MessageId (Member data 'MessageId))
+ '('FirstReceiveTimestamp receiveTimestamp)
+ '('LockTimestamp now)
+ '('ReceiveCount (Add (Member read 'ReceiveCount) (Uint32 '1)))
+ '('SentTimestamp (Member read 'SentTimestamp))
+ '('VisibilityDeadline (If valid (Member item 'VisibilityDeadline) (Member read 'VisibilityDeadline)))
+ '('Valid valid)
+ '('Exists (Exists read))))
+ )))))
+
+ (let validMessages
+ (Filter records (lambda '(item) (block '(
+ (return (Coalesce (Member item 'Valid) (Bool 'false))))))))
+
+ (let messagesToMove
+ (Filter validMessages (lambda '(item) (block '(
+ (return (Coalesce (Member item 'IsDeadLetter) (Bool 'false))))))))
+
+ (let messagesToUpdate
+ (Filter validMessages (lambda '(item) (block '(
+ (return (Coalesce (Not (Member item 'IsDeadLetter)) (Bool 'false))))))))
+
+ (let deadLetterStateRow '(
+ '('State (Uint64 '%4$lu))))
+ (let deadLetterStateSelect '(
+ 'MessageCount
+ 'WriteOffset
+ 'LastModifiedTimestamp))
+ (let deadLetterStateRead (SelectRow deadLetterStateTable deadLetterStateRow deadLetterStateSelect))
+
+ (let newDlqMessagesCount (Add (Member deadLetterStateRead 'MessageCount) (Length messagesToMove)))
+ (let newDlqWriteOffset (Add (Member deadLetterStateRead 'WriteOffset) deadLettersCount))
+ (let dlqStartOffset (Add (Member deadLetterStateRead 'WriteOffset) (Uint64 '1)))
+
+ (let dlqMostRecentTimestamp (Max (Member deadLetterStateRead 'LastModifiedTimestamp) now))
+
+ (let deadLetterStateUpdate '(
+ '('LastModifiedTimestamp dlqMostRecentTimestamp)
+ '('MessageCount newDlqMessagesCount)
+ '('WriteOffset newDlqWriteOffset)))
+
(let newSourceMessagesCount (Sub (Member sourceStateRead 'MessageCount) (Length messagesToMove)))
- (let sourceStateUpdate '(
- '('LastModifiedTimestamp (Max (Member sourceStateRead 'LastModifiedTimestamp) now))
+ (let sourceStateUpdate '(
+ '('LastModifiedTimestamp (Max (Member sourceStateRead 'LastModifiedTimestamp) now))
'('MessageCount newSourceMessagesCount)
- '('InflyCount (Sub (Member sourceStateRead 'InflyCount) (Length messagesToMove)))))
-
- (return (Extend
+ '('InflyCount (Sub (Member sourceStateRead 'InflyCount) (Length messagesToMove)))))
+
+ (return (Extend
(AsList (SetResult 'result records))
- (AsList (SetResult 'movedMessagesCount (Length messagesToMove)))
+ (AsList (SetResult 'movedMessagesCount (Length messagesToMove)))
(AsList (SetResult 'newMessagesCount newSourceMessagesCount))
- (AsList (UpdateRow deadLetterStateTable deadLetterStateRow deadLetterStateUpdate))
- (AsList (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate))
-
- (Map messagesToUpdate (lambda '(item) (block '(
- (let row '(
- '('Offset (Member item 'Offset))))
- (let update '(
- '('LoadId readId)
- '('FirstReceiveTimestamp (Member item 'FirstReceiveTimestamp))
- '('LockTimestamp (Member item 'LockTimestamp))
- '('ReceiveCount (Member item 'ReceiveCount))
- '('VisibilityDeadline (Member item 'VisibilityDeadline))))
- (return (UpdateRow sourceInflyTable row update))))))
-
- (Map messagesToMove (lambda '(item) (block '(
- (let msgRow '(
- '('Offset (Add dlqStartOffset (Member item 'DlqIndex)))))
- (let delayDeadline (Uint64 '0))
- (let messageUpdate '(
- '('RandomId readId)
- '('SentTimestamp dlqMostRecentTimestamp)
- '('DelayDeadline delayDeadline)))
- (return (UpdateRow deadLetterMessagesTable msgRow messageUpdate))))))
-
- (Map messagesToMove (lambda '(item) (block '(
- (let sentTsRow '(
- '('SentTimestamp dlqMostRecentTimestamp)
- '('Offset (Add dlqStartOffset (Member item 'DlqIndex)))))
- (let delayDeadline (Uint64 '0))
- (let sentTsUpdate '(
- '('RandomId readId)
- '('DelayDeadline delayDeadline)))
- (return (UpdateRow deadLetterSentTsIdxTable sentTsRow sentTsUpdate))))))
-
- (Map messagesToMove (lambda '(item) (block '(
- (let dataRow '(
- '('RandomId readId)
- '('Offset (Add dlqStartOffset (Member item 'DlqIndex)))))
-
- (let dataUpdate '(
- '('Data (Member item 'Data))
- '('Attributes (Member item 'Attributes))
- '('SenderId (Member item 'SenderId))
- '('MessageId (Member item 'MessageId))))
- (return (UpdateRow deadLetterMessageDataTable dataRow dataUpdate))))))
-
- (Map messagesToMove (lambda '(item) (block '(
- (let inflyRow '(
- '('Offset (Member item 'Offset))))
- (return (EraseRow sourceInflyTable inflyRow))))))
-
- (Map messagesToMove (lambda '(item) (block '(
- (let dataRow '(
- '('RandomId (Member item 'RandomId))
- '('Offset (Member item 'Offset))))
-
- (return (EraseRow sourceMessageDataTable dataRow))))))
-
- (Map messagesToMove (lambda '(item) (block '(
- (let sentTsRow '(
- '('SentTimestamp (Member item 'SentTimestamp))
- '('Offset (Member item 'Offset))))
- (return (EraseRow sourceSentTsIdxTable sentTsRow))))))
- ))
- )
-)__";
-
+ (AsList (UpdateRow deadLetterStateTable deadLetterStateRow deadLetterStateUpdate))
+ (AsList (UpdateRow sourceStateTable sourceStateRow sourceStateUpdate))
+
+ (Map messagesToUpdate (lambda '(item) (block '(
+ (let row '(
+ '('Offset (Member item 'Offset))))
+ (let update '(
+ '('LoadId readId)
+ '('FirstReceiveTimestamp (Member item 'FirstReceiveTimestamp))
+ '('LockTimestamp (Member item 'LockTimestamp))
+ '('ReceiveCount (Member item 'ReceiveCount))
+ '('VisibilityDeadline (Member item 'VisibilityDeadline))))
+ (return (UpdateRow sourceInflyTable row update))))))
+
+ (Map messagesToMove (lambda '(item) (block '(
+ (let msgRow '(
+ '('Offset (Add dlqStartOffset (Member item 'DlqIndex)))))
+ (let delayDeadline (Uint64 '0))
+ (let messageUpdate '(
+ '('RandomId readId)
+ '('SentTimestamp dlqMostRecentTimestamp)
+ '('DelayDeadline delayDeadline)))
+ (return (UpdateRow deadLetterMessagesTable msgRow messageUpdate))))))
+
+ (Map messagesToMove (lambda '(item) (block '(
+ (let sentTsRow '(
+ '('SentTimestamp dlqMostRecentTimestamp)
+ '('Offset (Add dlqStartOffset (Member item 'DlqIndex)))))
+ (let delayDeadline (Uint64 '0))
+ (let sentTsUpdate '(
+ '('RandomId readId)
+ '('DelayDeadline delayDeadline)))
+ (return (UpdateRow deadLetterSentTsIdxTable sentTsRow sentTsUpdate))))))
+
+ (Map messagesToMove (lambda '(item) (block '(
+ (let dataRow '(
+ '('RandomId readId)
+ '('Offset (Add dlqStartOffset (Member item 'DlqIndex)))))
+
+ (let dataUpdate '(
+ '('Data (Member item 'Data))
+ '('Attributes (Member item 'Attributes))
+ '('SenderId (Member item 'SenderId))
+ '('MessageId (Member item 'MessageId))))
+ (return (UpdateRow deadLetterMessageDataTable dataRow dataUpdate))))))
+
+ (Map messagesToMove (lambda '(item) (block '(
+ (let inflyRow '(
+ '('Offset (Member item 'Offset))))
+ (return (EraseRow sourceInflyTable inflyRow))))))
+
+ (Map messagesToMove (lambda '(item) (block '(
+ (let dataRow '(
+ '('RandomId (Member item 'RandomId))
+ '('Offset (Member item 'Offset))))
+
+ (return (EraseRow sourceMessageDataTable dataRow))))))
+
+ (Map messagesToMove (lambda '(item) (block '(
+ (let sentTsRow '(
+ '('SentTimestamp (Member item 'SentTimestamp))
+ '('Offset (Member item 'Offset))))
+ (return (EraseRow sourceSentTsIdxTable sentTsRow))))))
+ ))
+ )
+)__";
+
const char* const WriteMessageQuery = R"__(
(
(let randomId (Parameter 'RANDOM_ID (DataType 'Uint64)))
@@ -1031,7 +1031,7 @@ const char* const LoadInflyQuery = R"__(
'Offset
'RandomId
'DelayDeadline
- 'ReceiveCount
+ 'ReceiveCount
'VisibilityDeadline))
(let infly (Member (SelectRange inflyTable range fields '()) 'List))
@@ -1069,53 +1069,53 @@ const char* const GetStateQuery = R"__(
)
)__";
-const char* const ListDeadLetterSourceQueuesQuery = R"__(
- (
+const char* const ListDeadLetterSourceQueuesQuery = R"__(
+ (
(let folderId (Parameter 'FOLDERID (DataType 'Utf8String)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
-
+
(let queuesTable '%5$s/.Queues)
(let queuesRow '(
'('Account userName)
- '('QueueName (Utf8String '"%4$s"))))
-
+ '('QueueName (Utf8String '"%4$s"))))
+
(let queuesRowSelect '(
- 'QueueName
- 'CustomQueueName))
-
+ 'QueueName
+ 'CustomQueueName))
+
(let queuesRowRead (SelectRow queuesTable queuesRow queuesRowSelect))
-
- (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
-
- (let dlqName
+
+ (let skipFolderIdFilter (Equal folderId (Utf8String '"")))
+
+ (let dlqName
(If skipFolderIdFilter (Member queuesRowRead 'QueueName) (Coalesce (Member queuesRowRead 'CustomQueueName) (Utf8String '""))))
-
+
(let queuesRange '(
'('Account userName userName)
- '('QueueName (Utf8String '"") (Void))))
- (let queueSelect '('QueueName 'QueueState 'FolderId 'DlqName 'CustomQueueName))
+ '('QueueName (Utf8String '"") (Void))))
+ (let queueSelect '('QueueName 'QueueState 'FolderId 'DlqName 'CustomQueueName))
(let queues (Member (SelectRange queuesTable queuesRange queueSelect '()) 'List))
-
- (let filtered (Filter queues (lambda '(item) (block '(
- (return (Coalesce
- (And
- (And
- (Or
- (Equal (Member item 'QueueState) (Uint64 '1))
- (Equal (Member item 'QueueState) (Uint64 '3)))
- (Or
- (Equal (Member item 'FolderId) folderId)
- skipFolderIdFilter))
- (Equal (Member item 'DlqName) dlqName))
- (Bool 'false)))
- )))))
-
- (return (AsList
- (SetResult 'queues filtered)))
- )
-)__";
-
+
+ (let filtered (Filter queues (lambda '(item) (block '(
+ (return (Coalesce
+ (And
+ (And
+ (Or
+ (Equal (Member item 'QueueState) (Uint64 '1))
+ (Equal (Member item 'QueueState) (Uint64 '3)))
+ (Or
+ (Equal (Member item 'FolderId) folderId)
+ skipFolderIdFilter))
+ (Equal (Member item 'DlqName) dlqName))
+ (Bool 'false)))
+ )))))
+
+ (return (AsList
+ (SetResult 'queues filtered)))
+ )
+)__";
+
const char* const GetUserSettingsQuery = R"__(
(
(let fromUser (Parameter 'FROM_USER (DataType 'Utf8String)))
@@ -1142,29 +1142,29 @@ const char* const GetUserSettingsQuery = R"__(
)
)__";
-const char* const GetMessageCountMetricsQuery = R"__(
- (
- (let shard (Parameter 'SHARD (DataType 'Uint64)))
-
- (let stateTable '%1$s/State)
-
- (let stateRow '(
- '('State shard)))
- (let stateSelect '(
- 'MessageCount
- 'InflyCount
- 'CreatedTimestamp))
-
- (let stateRead
- (SelectRow stateTable stateRow stateSelect))
-
- (return (AsList
- (SetResult 'messagesCount (Member stateRead 'MessageCount))
- (SetResult 'inflyMessagesCount (Member stateRead 'InflyCount))
- (SetResult 'createdTimestamp (Member stateRead 'CreatedTimestamp))))
- )
-)__";
-
+const char* const GetMessageCountMetricsQuery = R"__(
+ (
+ (let shard (Parameter 'SHARD (DataType 'Uint64)))
+
+ (let stateTable '%1$s/State)
+
+ (let stateRow '(
+ '('State shard)))
+ (let stateSelect '(
+ 'MessageCount
+ 'InflyCount
+ 'CreatedTimestamp))
+
+ (let stateRead
+ (SelectRow stateTable stateRow stateSelect))
+
+ (return (AsList
+ (SetResult 'messagesCount (Member stateRead 'MessageCount))
+ (SetResult 'inflyMessagesCount (Member stateRead 'InflyCount))
+ (SetResult 'createdTimestamp (Member stateRead 'CreatedTimestamp))))
+ )
+)__";
+
const char* const GetQueuesListQuery = R"__(
(
(let fromUser (Parameter 'FROM_USER (DataType 'Utf8String)))
@@ -1182,7 +1182,7 @@ const char* const GetQueuesListQuery = R"__(
'QueueState
'CreatedTimestamp
'CustomQueueName
- 'DlqName
+ 'DlqName
'FolderId
'MasterTabletId
'Version
@@ -1232,16 +1232,16 @@ const char* GetStdQueryById(size_t id) {
return AddMessagesToInflyQuery;
case GET_STATE_ID: // 19
return GetStateQuery;
- case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 20
- return ListDeadLetterSourceQueuesQuery;
- case LOAD_OR_REDRIVE_MESSAGE_ID: // 21
- return LoadOrRedriveMessageQuery;
+ case LIST_DEAD_LETTER_SOURCE_QUEUES_ID: // 20
+ return ListDeadLetterSourceQueuesQuery;
+ case LOAD_OR_REDRIVE_MESSAGE_ID: // 21
+ return LoadOrRedriveMessageQuery;
case GET_USER_SETTINGS_ID: // 23
return GetUserSettingsQuery;
case GET_QUEUES_LIST_ID: // 24
return GetQueuesListQuery;
- case GET_MESSAGE_COUNT_METRIC_ID: // 14
- return GetMessageCountMetricsQuery;
+ case GET_MESSAGE_COUNT_METRIC_ID: // 14
+ return GetMessageCountMetricsQuery;
}
return nullptr;
}
diff --git a/ydb/core/ymq/queues/std/schema.cpp b/ydb/core/ymq/queues/std/schema.cpp
index 806a2f1d7f..4ba75d17a2 100644
--- a/ydb/core/ymq/queues/std/schema.cpp
+++ b/ydb/core/ymq/queues/std/schema.cpp
@@ -11,9 +11,9 @@ TVector<TTable> GetStandardTables(ui64 shards, ui64 partitions, bool enableAutos
TColumn("MaximumMessageSize", NScheme::NTypeIds::Uint64),
TColumn("MessageRetentionPeriod", NScheme::NTypeIds::Uint64),
TColumn("ReceiveMessageWaitTime", NScheme::NTypeIds::Uint64),
- TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64),
- TColumn("DlqName", NScheme::NTypeIds::Utf8),
- TColumn("DlqArn", NScheme::NTypeIds::Utf8),
+ TColumn("VisibilityTimeout", NScheme::NTypeIds::Uint64),
+ TColumn("DlqName", NScheme::NTypeIds::Utf8),
+ TColumn("DlqArn", NScheme::NTypeIds::Utf8),
TColumn("MaxReceiveCount", NScheme::NTypeIds::Uint64),
TColumn("ShowDetailedCountersDeadline", NScheme::NTypeIds::Uint64)};
@@ -35,8 +35,8 @@ TVector<TTable> GetStandardTables(ui64 shards, ui64 partitions, bool enableAutos
TColumn("Offset", NScheme::NTypeIds::Uint64, true),
TColumn("Attributes", NScheme::NTypeIds::String),
TColumn("Data", NScheme::NTypeIds::String),
- TColumn("MessageId", NScheme::NTypeIds::String),
- TColumn("SenderId", NScheme::NTypeIds::String)};
+ TColumn("MessageId", NScheme::NTypeIds::String),
+ TColumn("SenderId", NScheme::NTypeIds::String)};
const TVector<TColumn> MessagesColumns = {
TColumn("Offset", NScheme::NTypeIds::Uint64, true),
diff --git a/ydb/core/ymq/ut/queue_id_ut.cpp b/ydb/core/ymq/ut/queue_id_ut.cpp
index 0ac3776df6..6e7305fc82 100644
--- a/ydb/core/ymq/ut/queue_id_ut.cpp
+++ b/ydb/core/ymq/ut/queue_id_ut.cpp
@@ -1,28 +1,28 @@
#include <library/cpp/testing/unittest/registar.h>
#include <ydb/core/ymq/base/queue_id.h>
-
-using namespace NKikimr::NSQS;
-
-Y_UNIT_TEST_SUITE(TGenerateQueueIdTests) {
- static TString ValidateAndReturn(const TString& resourceId) {
- UNIT_ASSERT_EQUAL(resourceId.size(), 20);
-
- for (const auto c : resourceId) {
- UNIT_ASSERT(c <= 'v' && c >= '0');
- }
-
- return resourceId;
- }
-
- Y_UNIT_TEST(MakeQueueIdBasic) {
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 0, "radix"), ValidateAndReturn("03r0000000000000024u"));
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 1, "radix"), ValidateAndReturn("03r0000000000001024u"));
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 1, "radix"), ValidateAndReturn("vvv0000000000001024u"));
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("000fvvvvvvvvvvvv024u"));
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("vvvfvvvvvvvvvvvv024u"));
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0, "Man, it is a very long account name, but does anyone really care?"), ValidateAndReturn("000000000000000006at"));
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 16, "radix"), ValidateAndReturn("03r000000000000g024u"));
- UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0x097778AD59B52C00, "radix"), ValidateAndReturn("0000itrollcrab00024u"));
- }
-
-}
+
+using namespace NKikimr::NSQS;
+
+Y_UNIT_TEST_SUITE(TGenerateQueueIdTests) {
+ static TString ValidateAndReturn(const TString& resourceId) {
+ UNIT_ASSERT_EQUAL(resourceId.size(), 20);
+
+ for (const auto c : resourceId) {
+ UNIT_ASSERT(c <= 'v' && c >= '0');
+ }
+
+ return resourceId;
+ }
+
+ Y_UNIT_TEST(MakeQueueIdBasic) {
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 0, "radix"), ValidateAndReturn("03r0000000000000024u"));
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 1, "radix"), ValidateAndReturn("03r0000000000001024u"));
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 1, "radix"), ValidateAndReturn("vvv0000000000001024u"));
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("000fvvvvvvvvvvvv024u"));
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0x7FFF, 0xFFFFFFFFFFFFFFFF, "radix"), ValidateAndReturn("vvvfvvvvvvvvvvvv024u"));
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0, "Man, it is a very long account name, but does anyone really care?"), ValidateAndReturn("000000000000000006at"));
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(123, 16, "radix"), ValidateAndReturn("03r000000000000g024u"));
+ UNIT_ASSERT_STRINGS_EQUAL(MakeQueueId(0, 0x097778AD59B52C00, "radix"), ValidateAndReturn("0000itrollcrab00024u"));
+ }
+
+}
diff --git a/ydb/core/ymq/ut/ya.make b/ydb/core/ymq/ut/ya.make
index 112a1b60bb..27db5cd822 100644
--- a/ydb/core/ymq/ut/ya.make
+++ b/ydb/core/ymq/ut/ya.make
@@ -6,7 +6,7 @@ OWNER(
UNITTEST()
SRCS(
- queue_id_ut.cpp
+ queue_id_ut.cpp
params_ut.cpp
)
diff --git a/ydb/library/aclib/aclib.cpp b/ydb/library/aclib/aclib.cpp
index 4d8d73370f..946497ea30 100644
--- a/ydb/library/aclib/aclib.cpp
+++ b/ydb/library/aclib/aclib.cpp
@@ -1,7 +1,7 @@
#include "aclib.h"
#include <util/stream/str.h>
#include <util/string/vector.h>
-#include <algorithm>
+#include <algorithm>
#include <util/string/split.h>
#include <library/cpp/protobuf/util/is_equal.h>
@@ -325,7 +325,7 @@ std::pair<ui32, ui32> TACL::ClearAccess() {
std::pair<ui32, ui32> TACL::RemoveAccess(const NACLibProto::TACE& filter) {
std::pair<ui32, ui32> modified = {};
- auto* ACL = MutableACE();
+ auto* ACL = MutableACE();
auto matchACE = [&filter](const NACLibProto::TACE& ace) {
if (filter.HasAccessType() && ace.GetAccessType() != filter.GetAccessType()) {
return false;
@@ -340,10 +340,10 @@ std::pair<ui32, ui32> TACL::RemoveAccess(const NACLibProto::TACE& filter) {
return false;
}
return true;
- };
-
+ };
+
auto newEnd = std::remove_if(ACL->begin(), ACL->end(), std::move(matchACE));
- while (newEnd != ACL->end()) {
+ while (newEnd != ACL->end()) {
switch (static_cast<NACLib::EAccessType>(ACL->rbegin()->GetAccessType())) {
case NACLib::EAccessType::Allow:
modified |= std::pair<ui32, ui32>(ACL->rbegin()->GetAccessRight(), NACLib::EAccessRights::NoAccess);
@@ -352,7 +352,7 @@ std::pair<ui32, ui32> TACL::RemoveAccess(const NACLibProto::TACE& filter) {
modified |= std::pair<ui32, ui32>(NACLib::EAccessRights::NoAccess, ACL->rbegin()->GetAccessRight());
break;
}
- ACL->RemoveLast();
+ ACL->RemoveLast();
}
return modified;
}
diff --git a/ydb/library/aclib/aclib_ut.cpp b/ydb/library/aclib/aclib_ut.cpp
index b6a00d11d1..c5828f8624 100644
--- a/ydb/library/aclib/aclib_ut.cpp
+++ b/ydb/library/aclib/aclib_ut.cpp
@@ -147,55 +147,55 @@ Y_UNIT_TEST_SUITE(ACLib) {
UNIT_ASSERT(effectiveAnimalFilesACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true);
UNIT_ASSERT(effectiveAnimalFilesACL.CheckAccess(EAccessRights::UpdateRow, dogToken) == true);
}
-
- Y_UNIT_TEST(TestDiffACL) {
- TUserToken jamesToken(James, TVector<TSID>());
- TUserToken catToken(Cat, TVector<TSID>());
+
+ Y_UNIT_TEST(TestDiffACL) {
+ TUserToken jamesToken(James, TVector<TSID>());
+ TUserToken catToken(Cat, TVector<TSID>());
TUserToken dogToken(Dog, TVector<TSID>());
-
- TSecurityObject objACL(James, true /* container */);
- TDiffACL addAccessDiff;
- addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer);
- addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer);
- addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer);
- addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer);
+
+ TSecurityObject objACL(James, true /* container */);
+ TDiffACL addAccessDiff;
+ addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer);
+ addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer);
+ addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer);
+ addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer);
addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::CreateQueue, Dog, EInheritanceType::InheritContainer);
addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::UpdateRow, Dog, EInheritanceType::InheritContainer);
addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::SelectRow, Dog, EInheritanceType::InheritContainer);
addAccessDiff.AddAccess(EAccessType::Allow, EAccessRights::AlterSchema, Dog, EInheritanceType::InheritContainer);
-
- objACL.ApplyDiff(addAccessDiff);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == true);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false);
-
- TDiffACL removeAccessDiff;
- removeAccessDiff.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer);
- objACL.ApplyDiff(removeAccessDiff);
-
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false);
-
- TDiffACL removeAccessDiff2;
- removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer);
- removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer);
- removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer);
- removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer);
- objACL.ApplyDiff(removeAccessDiff2);
-
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false);
- UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false);
+
+ objACL.ApplyDiff(addAccessDiff);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == true);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false);
+
+ TDiffACL removeAccessDiff;
+ removeAccessDiff.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer);
+ objACL.ApplyDiff(removeAccessDiff);
+
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == true);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == true);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == true);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false);
+
+ TDiffACL removeAccessDiff2;
+ removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::CreateQueue, Cat, EInheritanceType::InheritContainer);
+ removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::UpdateRow, Cat, EInheritanceType::InheritContainer);
+ removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::SelectRow, Cat, EInheritanceType::InheritContainer);
+ removeAccessDiff2.RemoveAccess(EAccessType::Allow, EAccessRights::AlterSchema, Cat, EInheritanceType::InheritContainer);
+ objACL.ApplyDiff(removeAccessDiff2);
+
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::ReadAttributes, catToken) == false);
+ UNIT_ASSERT(objACL.CheckAccess(EAccessRights::WriteAttributes, catToken) == false);
UNIT_ASSERT(objACL.CheckAccess(EAccessRights::CreateQueue, dogToken) == true);
UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, dogToken) == true);
@@ -210,7 +210,7 @@ Y_UNIT_TEST_SUITE(ACLib) {
UNIT_ASSERT(objACL.CheckAccess(EAccessRights::UpdateRow, dogToken) == false);
UNIT_ASSERT(objACL.CheckAccess(EAccessRights::SelectRow, dogToken) == false);
UNIT_ASSERT(objACL.CheckAccess(EAccessRights::AlterSchema, dogToken) == false);
- }
+ }
Y_UNIT_TEST(TestInhertACL) {
TSecurityObject objParent(James, true /* container */);
diff --git a/ydb/library/http_proxy/authorization/auth_helpers.cpp b/ydb/library/http_proxy/authorization/auth_helpers.cpp
index 178c658a5c..1d3cf30400 100644
--- a/ydb/library/http_proxy/authorization/auth_helpers.cpp
+++ b/ydb/library/http_proxy/authorization/auth_helpers.cpp
@@ -1,4 +1,4 @@
-#include "auth_helpers.h"
+#include "auth_helpers.h"
#include <ydb/library/http_proxy/error/error.h>
diff --git a/ydb/library/http_proxy/authorization/signature.cpp b/ydb/library/http_proxy/authorization/signature.cpp
index 5abef495d6..60368c0614 100644
--- a/ydb/library/http_proxy/authorization/signature.cpp
+++ b/ydb/library/http_proxy/authorization/signature.cpp
@@ -1,262 +1,262 @@
-#include "auth_helpers.h"
-#include "signature.h"
-
+#include "auth_helpers.h"
+#include "signature.h"
+
#include <library/cpp/http/io/stream.h>
#include <library/cpp/http/misc/parsed_request.h>
-
+
#include <contrib/libs/openssl/include/openssl/evp.h>
#include <contrib/libs/openssl/include/openssl/hmac.h>
#include <contrib/libs/openssl/include/openssl/sha.h>
-
-#include <util/generic/algorithm.h>
-#include <util/stream/str.h>
-#include <util/string/builder.h>
+
+#include <util/generic/algorithm.h>
+#include <util/stream/str.h>
+#include <util/string/builder.h>
#include <library/cpp/cgiparam/cgiparam.h>
-#include <util/string/hex.h>
-#include <util/string/join.h>
-#include <util/string/strip.h>
-
+#include <util/string/hex.h>
+#include <util/string/join.h>
+#include <util/string/strip.h>
+
namespace NKikimr::NSQS {
-
-static TString HmacSHA256(TStringBuf key, TStringBuf data) {
- unsigned char hash[SHA256_DIGEST_LENGTH];
- ui32 hl = SHA256_DIGEST_LENGTH;
+
+static TString HmacSHA256(TStringBuf key, TStringBuf data) {
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ ui32 hl = SHA256_DIGEST_LENGTH;
const auto* res = HMAC(EVP_sha256(), key.data(), key.size(), reinterpret_cast<const unsigned char*>(data.data()), data.size(), hash, &hl);
- Y_ENSURE(res);
- Y_ENSURE(hl == SHA256_DIGEST_LENGTH);
- return TString{reinterpret_cast<const char*>(res), hl};
-}
-
-static TString HashSHA256(IInputStream& stream) {
- SHA256_CTX hasher;
- SHA256_Init(&hasher);
- char buf[4096];
- size_t read = 0;
- while ((read = stream.Read(buf, sizeof(buf))) != 0) {
- SHA256_Update(&hasher, buf, read);
- }
- unsigned char hash[SHA256_DIGEST_LENGTH];
- SHA256_Final(hash, &hasher);
- return to_lower(HexEncode(hash, SHA256_DIGEST_LENGTH));
-}
-
-static TString HashSHA256(TStringBuf data) {
- TMemoryInput in{data};
- return HashSHA256(in);
-}
-
-static const char newline = '\n';
-static const char pathDelim = '/';
-
+ Y_ENSURE(res);
+ Y_ENSURE(hl == SHA256_DIGEST_LENGTH);
+ return TString{reinterpret_cast<const char*>(res), hl};
+}
+
+static TString HashSHA256(IInputStream& stream) {
+ SHA256_CTX hasher;
+ SHA256_Init(&hasher);
+ char buf[4096];
+ size_t read = 0;
+ while ((read = stream.Read(buf, sizeof(buf))) != 0) {
+ SHA256_Update(&hasher, buf, read);
+ }
+ unsigned char hash[SHA256_DIGEST_LENGTH];
+ SHA256_Final(hash, &hasher);
+ return to_lower(HexEncode(hash, SHA256_DIGEST_LENGTH));
+}
+
+static TString HashSHA256(TStringBuf data) {
+ TMemoryInput in{data};
+ return HashSHA256(in);
+}
+
+static const char newline = '\n';
+static const char pathDelim = '/';
+
constexpr TStringBuf AUTHORIZATION_HEADER = "authorization";
-
-static const TString SIGNED_HEADERS_PARAM = "signedheaders";
-static const TString SIGNATURE_PARAM = "signature";
-static const TString CREDENTIAL_PARAM = "credential";
-
-TAwsRequestSignV4::TAwsRequestSignV4(const TString& request) {
- // special standalone ctor for tests
- TStringInput si(request);
- THttpInput input(&si);
- TParsedHttpFull parsed(input.FirstLine());
-
- TMaybe<TBuffer> inputData;
- ui64 contentLength = 0;
- if (parsed.Method == "POST") {
- if (input.GetContentLength(contentLength)) {
- inputData.ConstructInPlace();
- inputData->Resize(contentLength);
- if (input.Load(inputData->Data(), (size_t)contentLength) != contentLength) {
- Y_VERIFY(false);
- }
- }
- }
-
- Process(input, parsed, inputData);
-}
-
-TAwsRequestSignV4::TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) {
- Process(input, parsed, inputData);
-}
-
-const TString& TAwsRequestSignV4::GetCanonicalRequest() const {
- return CanonicalRequestStr_;
-}
-
-const TString& TAwsRequestSignV4::GetStringToSign() const {
- return FinalStringToSignStr_;
-}
-
-const TString& TAwsRequestSignV4::GetParsedSignature() const {
- return ParsedSignature_;
-}
-
-const TString& TAwsRequestSignV4::GetSigningTimestamp() const {
- return AwsTimestamp_;
-}
-
-const TString& TAwsRequestSignV4::GetRegion() const {
- return AwsRegion_;
-}
-
-const TString& TAwsRequestSignV4::GetAccessKeyId() const {
- return AccessKeyId_;
-}
-
-void TAwsRequestSignV4::Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) {
- ParseAuthorization(input);
- MakeCanonicalRequest(input, parsed, inputData);
- MakeFinalStringToSign();
-}
-
-TString TAwsRequestSignV4::CalcSignature(const TString& secretKey) const {
- const auto dateKey = HmacSHA256(TString::Join("AWS4", secretKey), AwsDate_);
- const auto dateRegionKey = HmacSHA256(dateKey, AwsRegion_);
- const auto dateRegionServiceKey = HmacSHA256(dateRegionKey, AwsService_);
- const auto signingKey = HmacSHA256(dateRegionServiceKey, AwsRequest_);
- const auto signatureHmac = HmacSHA256(signingKey, FinalStringToSignStr_);
-
+
+static const TString SIGNED_HEADERS_PARAM = "signedheaders";
+static const TString SIGNATURE_PARAM = "signature";
+static const TString CREDENTIAL_PARAM = "credential";
+
+TAwsRequestSignV4::TAwsRequestSignV4(const TString& request) {
+ // special standalone ctor for tests
+ TStringInput si(request);
+ THttpInput input(&si);
+ TParsedHttpFull parsed(input.FirstLine());
+
+ TMaybe<TBuffer> inputData;
+ ui64 contentLength = 0;
+ if (parsed.Method == "POST") {
+ if (input.GetContentLength(contentLength)) {
+ inputData.ConstructInPlace();
+ inputData->Resize(contentLength);
+ if (input.Load(inputData->Data(), (size_t)contentLength) != contentLength) {
+ Y_VERIFY(false);
+ }
+ }
+ }
+
+ Process(input, parsed, inputData);
+}
+
+TAwsRequestSignV4::TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) {
+ Process(input, parsed, inputData);
+}
+
+const TString& TAwsRequestSignV4::GetCanonicalRequest() const {
+ return CanonicalRequestStr_;
+}
+
+const TString& TAwsRequestSignV4::GetStringToSign() const {
+ return FinalStringToSignStr_;
+}
+
+const TString& TAwsRequestSignV4::GetParsedSignature() const {
+ return ParsedSignature_;
+}
+
+const TString& TAwsRequestSignV4::GetSigningTimestamp() const {
+ return AwsTimestamp_;
+}
+
+const TString& TAwsRequestSignV4::GetRegion() const {
+ return AwsRegion_;
+}
+
+const TString& TAwsRequestSignV4::GetAccessKeyId() const {
+ return AccessKeyId_;
+}
+
+void TAwsRequestSignV4::Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) {
+ ParseAuthorization(input);
+ MakeCanonicalRequest(input, parsed, inputData);
+ MakeFinalStringToSign();
+}
+
+TString TAwsRequestSignV4::CalcSignature(const TString& secretKey) const {
+ const auto dateKey = HmacSHA256(TString::Join("AWS4", secretKey), AwsDate_);
+ const auto dateRegionKey = HmacSHA256(dateKey, AwsRegion_);
+ const auto dateRegionServiceKey = HmacSHA256(dateRegionKey, AwsService_);
+ const auto signingKey = HmacSHA256(dateRegionServiceKey, AwsRequest_);
+ const auto signatureHmac = HmacSHA256(signingKey, FinalStringToSignStr_);
+
return to_lower(HexEncode(signatureHmac.data(), signatureHmac.size()));
-}
-
-void TAwsRequestSignV4::ParseAuthorization(const THttpInput& input) {
- for (const auto& header : input.Headers()) {
- if (AsciiEqualsIgnoreCase(header.Name(), AUTHORIZATION_HEADER)) {
- auto params = ParseAuthorizationParams(header.Value());
-
- SignedHeadersList_ = TString(params[SIGNED_HEADERS_PARAM]);
- ParsedSignature_ = TString(params[SIGNATURE_PARAM]);
-
- TStringBuf credential = params[CREDENTIAL_PARAM];
- AccessKeyId_ = TString(credential.NextTok(pathDelim));
- AwsDate_ = TString(credential.NextTok(pathDelim));
- AwsRegion_ = TString(credential.NextTok(pathDelim));
- AwsService_ = TString(credential.NextTok(pathDelim));
- break;
- }
- }
-}
-
-static TString UriEncode(const TStringBuf input, bool encodeSlash = false) {
- TStringStream result;
- for (const char ch : input) {
- if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' ||
- ch == '-' || ch == '~' || ch == '.') {
- result << ch;
- } else if (ch == '/') {
- if (encodeSlash) {
- result << "%2F";
- } else {
- result << ch;
- }
- } else {
- result << "%" << HexEncode(&ch, 1);
- }
- }
- return result.Str();
-}
-
-void TAwsRequestSignV4::MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) {
- TStringStream canonicalRequest;
- // METHOD
- canonicalRequest << parsed.Method << newline;
- // PATH
- canonicalRequest << UriEncode(parsed.Path) << newline;
- // CGI
- TCgiParameters cgi(parsed.Cgi);
- TMap<TString, TVector<TString>> sortedCgi;
-
- for (const auto& [key, value] : cgi) {
- sortedCgi[key].push_back(value);
- }
-
- for (auto& pair : sortedCgi) {
- ::Sort(pair.second.begin(), pair.second.end());
- }
-
- if (sortedCgi.size()) {
- TStringStream canonicalCgi;
-
- auto printSingleParam = [&canonicalCgi](const TString& key, const TVector<TString>& values) {
- auto it = values.begin();
- canonicalCgi << UriEncode(key) << "=" << UriEncode(*it);
- while (++it != values.end()) {
- canonicalCgi << "&" << UriEncode(key) << "=" << UriEncode(*it);
- }
-
- };
-
- auto it = sortedCgi.begin();
- printSingleParam(it->first, it->second);
- while (++it != sortedCgi.end()) {
- canonicalCgi << "&";
- printSingleParam(it->first, it->second);
- }
-
- canonicalRequest << canonicalCgi.Str() << newline;
- } else {
- canonicalRequest << newline;
- }
- // CANONICAL HEADERS
- TMap<TStringBuf, TVector<TString>> canonicalHeaders;
-
- TStringBuf headersList = SignedHeadersList_;
- while (TStringBuf headerName = headersList.NextTok(';')) {
- canonicalHeaders[headerName] = {};
- }
-
- for (const auto& header : input.Headers()) {
- const auto lowercaseHeaderName = to_lower(header.Name());
- auto it = canonicalHeaders.find(lowercaseHeaderName);
- if (it != canonicalHeaders.end()) {
- it->second.push_back(Strip(header.Value()));
- }
- }
-
- for (const auto& [key, value] : canonicalHeaders) {
+}
+
+void TAwsRequestSignV4::ParseAuthorization(const THttpInput& input) {
+ for (const auto& header : input.Headers()) {
+ if (AsciiEqualsIgnoreCase(header.Name(), AUTHORIZATION_HEADER)) {
+ auto params = ParseAuthorizationParams(header.Value());
+
+ SignedHeadersList_ = TString(params[SIGNED_HEADERS_PARAM]);
+ ParsedSignature_ = TString(params[SIGNATURE_PARAM]);
+
+ TStringBuf credential = params[CREDENTIAL_PARAM];
+ AccessKeyId_ = TString(credential.NextTok(pathDelim));
+ AwsDate_ = TString(credential.NextTok(pathDelim));
+ AwsRegion_ = TString(credential.NextTok(pathDelim));
+ AwsService_ = TString(credential.NextTok(pathDelim));
+ break;
+ }
+ }
+}
+
+static TString UriEncode(const TStringBuf input, bool encodeSlash = false) {
+ TStringStream result;
+ for (const char ch : input) {
+ if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' ||
+ ch == '-' || ch == '~' || ch == '.') {
+ result << ch;
+ } else if (ch == '/') {
+ if (encodeSlash) {
+ result << "%2F";
+ } else {
+ result << ch;
+ }
+ } else {
+ result << "%" << HexEncode(&ch, 1);
+ }
+ }
+ return result.Str();
+}
+
+void TAwsRequestSignV4::MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData) {
+ TStringStream canonicalRequest;
+ // METHOD
+ canonicalRequest << parsed.Method << newline;
+ // PATH
+ canonicalRequest << UriEncode(parsed.Path) << newline;
+ // CGI
+ TCgiParameters cgi(parsed.Cgi);
+ TMap<TString, TVector<TString>> sortedCgi;
+
+ for (const auto& [key, value] : cgi) {
+ sortedCgi[key].push_back(value);
+ }
+
+ for (auto& pair : sortedCgi) {
+ ::Sort(pair.second.begin(), pair.second.end());
+ }
+
+ if (sortedCgi.size()) {
+ TStringStream canonicalCgi;
+
+ auto printSingleParam = [&canonicalCgi](const TString& key, const TVector<TString>& values) {
+ auto it = values.begin();
+ canonicalCgi << UriEncode(key) << "=" << UriEncode(*it);
+ while (++it != values.end()) {
+ canonicalCgi << "&" << UriEncode(key) << "=" << UriEncode(*it);
+ }
+
+ };
+
+ auto it = sortedCgi.begin();
+ printSingleParam(it->first, it->second);
+ while (++it != sortedCgi.end()) {
+ canonicalCgi << "&";
+ printSingleParam(it->first, it->second);
+ }
+
+ canonicalRequest << canonicalCgi.Str() << newline;
+ } else {
+ canonicalRequest << newline;
+ }
+ // CANONICAL HEADERS
+ TMap<TStringBuf, TVector<TString>> canonicalHeaders;
+
+ TStringBuf headersList = SignedHeadersList_;
+ while (TStringBuf headerName = headersList.NextTok(';')) {
+ canonicalHeaders[headerName] = {};
+ }
+
+ for (const auto& header : input.Headers()) {
+ const auto lowercaseHeaderName = to_lower(header.Name());
+ auto it = canonicalHeaders.find(lowercaseHeaderName);
+ if (it != canonicalHeaders.end()) {
+ it->second.push_back(Strip(header.Value()));
+ }
+ }
+
+ for (const auto& [key, value] : canonicalHeaders) {
canonicalRequest << key << ":"sv << JoinRange(",", value.begin(), value.end()) << newline;
- }
-
- canonicalRequest << newline; // skip additional line after headers
- // SIGNED HEADERS
- canonicalRequest << SignedHeadersList_ << newline;
-
- static const TStringBuf hashedContentHeader = "x-amz-content-sha256";
-
- // PAYLOAD
- auto contentIt = canonicalHeaders.find(hashedContentHeader);
- static const TStringBuf unsignedPayloadLiteral = "UNSIGNED-PAYLOAD";
- if (contentIt != canonicalHeaders.end() && !contentIt->second.empty() && contentIt->second[0] == unsignedPayloadLiteral) {
- canonicalRequest << unsignedPayloadLiteral;
- } else {
- if (inputData) {
- canonicalRequest << HashSHA256(TStringBuf(inputData->Data(), inputData->Size()));
- } else {
- static const TStringBuf emptyStringSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
- canonicalRequest << emptyStringSHA256;
- }
- }
-
- auto amzDateIt = canonicalHeaders.find("x-amz-date");
- if (amzDateIt != canonicalHeaders.end() && !amzDateIt->second.empty()) {
- AwsTimestamp_ = amzDateIt->second[0];
- }
-
- CanonicalRequestStr_ = canonicalRequest.Str();
-}
-
-void TAwsRequestSignV4::MakeFinalStringToSign() {
- TStringStream finalStringToSign;
+ }
+
+ canonicalRequest << newline; // skip additional line after headers
+ // SIGNED HEADERS
+ canonicalRequest << SignedHeadersList_ << newline;
+
+ static const TStringBuf hashedContentHeader = "x-amz-content-sha256";
+
+ // PAYLOAD
+ auto contentIt = canonicalHeaders.find(hashedContentHeader);
+ static const TStringBuf unsignedPayloadLiteral = "UNSIGNED-PAYLOAD";
+ if (contentIt != canonicalHeaders.end() && !contentIt->second.empty() && contentIt->second[0] == unsignedPayloadLiteral) {
+ canonicalRequest << unsignedPayloadLiteral;
+ } else {
+ if (inputData) {
+ canonicalRequest << HashSHA256(TStringBuf(inputData->Data(), inputData->Size()));
+ } else {
+ static const TStringBuf emptyStringSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+ canonicalRequest << emptyStringSHA256;
+ }
+ }
+
+ auto amzDateIt = canonicalHeaders.find("x-amz-date");
+ if (amzDateIt != canonicalHeaders.end() && !amzDateIt->second.empty()) {
+ AwsTimestamp_ = amzDateIt->second[0];
+ }
+
+ CanonicalRequestStr_ = canonicalRequest.Str();
+}
+
+void TAwsRequestSignV4::MakeFinalStringToSign() {
+ TStringStream finalStringToSign;
finalStringToSign << "AWS4-HMAC-SHA256"sv << newline;
- finalStringToSign << AwsTimestamp_ << newline;
-
- finalStringToSign << AwsDate_ << pathDelim;
- finalStringToSign << AwsRegion_ << pathDelim;
- finalStringToSign << AwsService_ << pathDelim;
- finalStringToSign << AwsRequest_ << newline;
-
- finalStringToSign << HashSHA256(CanonicalRequestStr_);
- FinalStringToSignStr_ = finalStringToSign.Str();
-}
-
+ finalStringToSign << AwsTimestamp_ << newline;
+
+ finalStringToSign << AwsDate_ << pathDelim;
+ finalStringToSign << AwsRegion_ << pathDelim;
+ finalStringToSign << AwsService_ << pathDelim;
+ finalStringToSign << AwsRequest_ << newline;
+
+ finalStringToSign << HashSHA256(CanonicalRequestStr_);
+ FinalStringToSignStr_ = finalStringToSign.Str();
+}
+
} // namespace NKikimr::NSQS
diff --git a/ydb/library/http_proxy/authorization/signature.h b/ydb/library/http_proxy/authorization/signature.h
index 4e65519d14..57468deee6 100644
--- a/ydb/library/http_proxy/authorization/signature.h
+++ b/ydb/library/http_proxy/authorization/signature.h
@@ -1,52 +1,52 @@
-#pragma once
-
-#include <util/generic/maybe.h>
-#include <util/generic/buffer.h>
-#include <util/generic/string.h>
-#include <util/stream/mem.h>
-
-class THttpInput;
-struct TParsedHttpFull;
-
+#pragma once
+
+#include <util/generic/maybe.h>
+#include <util/generic/buffer.h>
+#include <util/generic/string.h>
+#include <util/stream/mem.h>
+
+class THttpInput;
+struct TParsedHttpFull;
+
namespace NKikimr::NSQS {
-
-class TAwsRequestSignV4 {
-public:
- TAwsRequestSignV4(const TString& request);
- TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData);
-
- const TString& GetCanonicalRequest() const;
-
- const TString& GetStringToSign() const;
-
- const TString& GetParsedSignature() const;
-
- const TString& GetSigningTimestamp() const;
-
- const TString& GetRegion() const;
-
- const TString& GetAccessKeyId() const;
-
- TString CalcSignature(const TString& secretKey) const;
-
-private:
- void Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData);
- void ParseAuthorization(const THttpInput& input);
- void MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData);
- void MakeFinalStringToSign();
-
-private:
- TString SignedHeadersList_;
- TString ParsedSignature_;
- TString CanonicalRequestStr_;
- TString FinalStringToSignStr_;
- TString StringToSign_;
- TString AwsTimestamp_; // e. g. 20130524T000000Z
- TString AccessKeyId_;
- TString AwsDate_; // YYYYMMDD
- TString AwsRegion_;
- TString AwsService_;
- TString AwsRequest_ = "aws4_request";
-};
-
+
+class TAwsRequestSignV4 {
+public:
+ TAwsRequestSignV4(const TString& request);
+ TAwsRequestSignV4(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData);
+
+ const TString& GetCanonicalRequest() const;
+
+ const TString& GetStringToSign() const;
+
+ const TString& GetParsedSignature() const;
+
+ const TString& GetSigningTimestamp() const;
+
+ const TString& GetRegion() const;
+
+ const TString& GetAccessKeyId() const;
+
+ TString CalcSignature(const TString& secretKey) const;
+
+private:
+ void Process(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData);
+ void ParseAuthorization(const THttpInput& input);
+ void MakeCanonicalRequest(const THttpInput& input, const TParsedHttpFull& parsed, const TMaybe<TBuffer>& inputData);
+ void MakeFinalStringToSign();
+
+private:
+ TString SignedHeadersList_;
+ TString ParsedSignature_;
+ TString CanonicalRequestStr_;
+ TString FinalStringToSignStr_;
+ TString StringToSign_;
+ TString AwsTimestamp_; // e. g. 20130524T000000Z
+ TString AccessKeyId_;
+ TString AwsDate_; // YYYYMMDD
+ TString AwsRegion_;
+ TString AwsService_;
+ TString AwsRequest_ = "aws4_request";
+};
+
} // namespace NKikimr::NSQS
diff --git a/ydb/library/http_proxy/authorization/ut/signature_ut.cpp b/ydb/library/http_proxy/authorization/ut/signature_ut.cpp
index e193ad3837..062ac52146 100644
--- a/ydb/library/http_proxy/authorization/ut/signature_ut.cpp
+++ b/ydb/library/http_proxy/authorization/ut/signature_ut.cpp
@@ -2,112 +2,112 @@
#include <library/cpp/testing/unittest/registar.h>
#include <ydb/library/http_proxy/error/error.h>
-
-using namespace NKikimr::NSQS;
-
-static const TString superSecretAmazonKey = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY";
-
-Y_UNIT_TEST_SUITE(TestAwsSignatureV4) {
- Y_UNIT_TEST(TestGetVanilla) {
- const TString request = \
-R"__(GET / HTTP/1.1
-Host:example.amazonaws.com
-X-Amz-Date:20150830T123600Z
-Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31)__";
-
- const TString canonicalizedRequest = \
-R"__(GET
-/
-
-host:example.amazonaws.com
-x-amz-date:20150830T123600Z
-
-host;x-amz-date
-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855)__";
-
- const TString stringToSign = \
-R"__(AWS4-HMAC-SHA256
-20150830T123600Z
-20150830/us-east-1/service/aws4_request
-bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63)__";
-
- TAwsRequestSignV4 signature(request);
- UNIT_ASSERT_EQUAL(canonicalizedRequest, signature.GetCanonicalRequest());
- UNIT_ASSERT_EQUAL(stringToSign, signature.GetStringToSign());
- UNIT_ASSERT_EQUAL("us-east-1", signature.GetRegion());
- UNIT_ASSERT_EQUAL("5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31", signature.CalcSignature(superSecretAmazonKey));
- }
-
- Y_UNIT_TEST(TestPostVanilla) {
- const TString request = \
-R"__(POST / HTTP/1.1
-Host:example.amazonaws.com
-X-Amz-Date:20150830T123600Z
-Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b)__";
- TAwsRequestSignV4 signature(request);
- UNIT_ASSERT_EQUAL("5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b", signature.CalcSignature(superSecretAmazonKey));
- }
-
- Y_UNIT_TEST(TestPostHeaderCase) {
- const TString request = \
-R"__(POST / HTTP/1.1
-Host:example.amazonaws.com
-My-Header1:VALUE1
-X-Amz-Date:20150830T123600Z
-Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d)__";
- TAwsRequestSignV4 signature(request);
- UNIT_ASSERT_EQUAL("cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d", signature.CalcSignature(superSecretAmazonKey));
- }
-
- Y_UNIT_TEST(TestGetParamsOrder) {
- const TString request = \
-R"__(GET /?Param2=value2&Param1=value1 HTTP/1.1
-Host:example.amazonaws.com
-X-Amz-Date:20150830T123600Z
-Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500)__";
- TAwsRequestSignV4 signature(request);
- UNIT_ASSERT_EQUAL("b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500", signature.CalcSignature(superSecretAmazonKey));
- }
-
- Y_UNIT_TEST(TestGetUtf8) {
- const TString request = \
-R"__(GET /?ሴ=bar HTTP/1.1
-Host:example.amazonaws.com
-X-Amz-Date:20150830T123600Z
-Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04)__";
- TAwsRequestSignV4 signature(request);
- UNIT_ASSERT_EQUAL("2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04", signature.CalcSignature(superSecretAmazonKey));
- }
-
- Y_UNIT_TEST(TestQueryOrderValue) {
- const TString request = \
-R"__(GET /?Param1=value2&Param1=value1 HTTP/1.1
-Host:example.amazonaws.com
-X-Amz-Date:20150830T123600Z
-Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694)__";
- TAwsRequestSignV4 signature(request);
- UNIT_ASSERT_EQUAL("5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694", signature.CalcSignature(superSecretAmazonKey));
- }
-
- Y_UNIT_TEST(TestGetHeaderKeyDuplicate) {
- const TString request = \
-R"__(GET / HTTP/1.1
-Host:example.amazonaws.com
-My-Header1:value4
-My-Header1:value1
-My-Header1:value3
-My-Header1:value2
-X-Amz-Date:20150830T123600Z
-Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01)__";
- TAwsRequestSignV4 signature(request);
- UNIT_ASSERT_EQUAL("08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01", signature.CalcSignature(superSecretAmazonKey));
- }
-
- Y_UNIT_TEST(TestEmptyReq) {
- const TString request = \
-R"__(GET /?a=b HTTP/1.1)__";
- TAwsRequestSignV4 signature(request);
- // this signature is totally incorrect, we only test the fact that the code is stable
- UNIT_ASSERT_EQUAL("5b63431f16d95e041cd7507fd76cbc072a1de46c09452621b8f25f1193997c2a", signature.CalcSignature(superSecretAmazonKey));
- }
-}
+
+using namespace NKikimr::NSQS;
+
+static const TString superSecretAmazonKey = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY";
+
+Y_UNIT_TEST_SUITE(TestAwsSignatureV4) {
+ Y_UNIT_TEST(TestGetVanilla) {
+ const TString request = \
+R"__(GET / HTTP/1.1
+Host:example.amazonaws.com
+X-Amz-Date:20150830T123600Z
+Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31)__";
+
+ const TString canonicalizedRequest = \
+R"__(GET
+/
+
+host:example.amazonaws.com
+x-amz-date:20150830T123600Z
+
+host;x-amz-date
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855)__";
+
+ const TString stringToSign = \
+R"__(AWS4-HMAC-SHA256
+20150830T123600Z
+20150830/us-east-1/service/aws4_request
+bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63)__";
+
+ TAwsRequestSignV4 signature(request);
+ UNIT_ASSERT_EQUAL(canonicalizedRequest, signature.GetCanonicalRequest());
+ UNIT_ASSERT_EQUAL(stringToSign, signature.GetStringToSign());
+ UNIT_ASSERT_EQUAL("us-east-1", signature.GetRegion());
+ UNIT_ASSERT_EQUAL("5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31", signature.CalcSignature(superSecretAmazonKey));
+ }
+
+ Y_UNIT_TEST(TestPostVanilla) {
+ const TString request = \
+R"__(POST / HTTP/1.1
+Host:example.amazonaws.com
+X-Amz-Date:20150830T123600Z
+Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b)__";
+ TAwsRequestSignV4 signature(request);
+ UNIT_ASSERT_EQUAL("5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b", signature.CalcSignature(superSecretAmazonKey));
+ }
+
+ Y_UNIT_TEST(TestPostHeaderCase) {
+ const TString request = \
+R"__(POST / HTTP/1.1
+Host:example.amazonaws.com
+My-Header1:VALUE1
+X-Amz-Date:20150830T123600Z
+Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d)__";
+ TAwsRequestSignV4 signature(request);
+ UNIT_ASSERT_EQUAL("cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d", signature.CalcSignature(superSecretAmazonKey));
+ }
+
+ Y_UNIT_TEST(TestGetParamsOrder) {
+ const TString request = \
+R"__(GET /?Param2=value2&Param1=value1 HTTP/1.1
+Host:example.amazonaws.com
+X-Amz-Date:20150830T123600Z
+Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500)__";
+ TAwsRequestSignV4 signature(request);
+ UNIT_ASSERT_EQUAL("b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500", signature.CalcSignature(superSecretAmazonKey));
+ }
+
+ Y_UNIT_TEST(TestGetUtf8) {
+ const TString request = \
+R"__(GET /?ሴ=bar HTTP/1.1
+Host:example.amazonaws.com
+X-Amz-Date:20150830T123600Z
+Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04)__";
+ TAwsRequestSignV4 signature(request);
+ UNIT_ASSERT_EQUAL("2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04", signature.CalcSignature(superSecretAmazonKey));
+ }
+
+ Y_UNIT_TEST(TestQueryOrderValue) {
+ const TString request = \
+R"__(GET /?Param1=value2&Param1=value1 HTTP/1.1
+Host:example.amazonaws.com
+X-Amz-Date:20150830T123600Z
+Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694)__";
+ TAwsRequestSignV4 signature(request);
+ UNIT_ASSERT_EQUAL("5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694", signature.CalcSignature(superSecretAmazonKey));
+ }
+
+ Y_UNIT_TEST(TestGetHeaderKeyDuplicate) {
+ const TString request = \
+R"__(GET / HTTP/1.1
+Host:example.amazonaws.com
+My-Header1:value4
+My-Header1:value1
+My-Header1:value3
+My-Header1:value2
+X-Amz-Date:20150830T123600Z
+Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01)__";
+ TAwsRequestSignV4 signature(request);
+ UNIT_ASSERT_EQUAL("08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01", signature.CalcSignature(superSecretAmazonKey));
+ }
+
+ Y_UNIT_TEST(TestEmptyReq) {
+ const TString request = \
+R"__(GET /?a=b HTTP/1.1)__";
+ TAwsRequestSignV4 signature(request);
+ // this signature is totally incorrect, we only test the fact that the code is stable
+ UNIT_ASSERT_EQUAL("5b63431f16d95e041cd7507fd76cbc072a1de46c09452621b8f25f1193997c2a", signature.CalcSignature(superSecretAmazonKey));
+ }
+}
diff --git a/ydb/library/http_proxy/error/error.cpp b/ydb/library/http_proxy/error/error.cpp
index 93e3756c9b..0c91aac859 100644
--- a/ydb/library/http_proxy/error/error.cpp
+++ b/ydb/library/http_proxy/error/error.cpp
@@ -191,12 +191,12 @@ extern const TErrorClass INVALID_ATTRIBUTE_NAME = {
"The specified attribute doesn't exist."
};
-extern const TErrorClass INVALID_ATTRIBUTE_VALUE = {
- "InvalidAttributeValue",
- 400,
- "The specified attribute value is invalid."
-};
-
+extern const TErrorClass INVALID_ATTRIBUTE_VALUE = {
+ "InvalidAttributeValue",
+ 400,
+ "The specified attribute value is invalid."
+};
+
extern const TErrorClass LEADER_RESOLVING_ERROR = {
"InternalFailure",
500,
diff --git a/ydb/library/http_proxy/error/error.h b/ydb/library/http_proxy/error/error.h
index 9845778b0e..02b9dca3d9 100644
--- a/ydb/library/http_proxy/error/error.h
+++ b/ydb/library/http_proxy/error/error.h
@@ -84,7 +84,7 @@ extern const TErrorClass RECEIPT_HANDLE_IS_INVALID;
// GetQueueAttributes errors
// https://docs.aws.amazon.com/en_us/AWSSimpleQueueService/latest/APIReference/API_GetQueueAttributes.html
extern const TErrorClass INVALID_ATTRIBUTE_NAME;
-extern const TErrorClass INVALID_ATTRIBUTE_VALUE;
+extern const TErrorClass INVALID_ATTRIBUTE_VALUE;
// Leader resolving errors
extern const TErrorClass LEADER_RESOLVING_ERROR;
diff --git a/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto b/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto
index 34449dde05..873ee38b3c 100644
--- a/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto
+++ b/ydb/public/api/grpc/draft/ydb_persqueue_v1.proto
@@ -1,5 +1,5 @@
syntax = "proto3";
-option cc_enable_arenas = true;
+option cc_enable_arenas = true;
package Ydb.PersQueue.V1;
@@ -103,9 +103,9 @@ service PersQueueService {
* Remove read rule command.
*/
rpc RemoveReadRule(RemoveReadRuleRequest) returns (RemoveReadRuleResponse);
-}
+}
-service ClusterDiscoveryService {
- // Get PQ clusters which are eligible for the specified Write or Read Sessions
- rpc DiscoverClusters(Ydb.PersQueue.ClusterDiscovery.DiscoverClustersRequest) returns (Ydb.PersQueue.ClusterDiscovery.DiscoverClustersResponse);
-}
+service ClusterDiscoveryService {
+ // Get PQ clusters which are eligible for the specified Write or Read Sessions
+ rpc DiscoverClusters(Ydb.PersQueue.ClusterDiscovery.DiscoverClustersRequest) returns (Ydb.PersQueue.ClusterDiscovery.DiscoverClustersResponse);
+}
diff --git a/ydb/public/api/protos/ya.make b/ydb/public/api/protos/ya.make
index 265061f17b..8a156403a3 100644
--- a/ydb/public/api/protos/ya.make
+++ b/ydb/public/api/protos/ya.make
@@ -23,7 +23,7 @@ SRCS(
persqueue_error_codes_v1.proto
ydb_auth.proto
ydb_persqueue_v1.proto
- ydb_persqueue_cluster_discovery.proto
+ ydb_persqueue_cluster_discovery.proto
ydb_clickhouse_internal.proto
ydb_cms.proto
ydb_common.proto
@@ -52,7 +52,7 @@ CPP_PROTO_PLUGIN0(validation ydb/core/grpc_services/validation)
# .pb.h are only available in C++ variant of PROTO_LIBRARY
IF (MODULE_TAG == "CPP_PROTO")
GENERATE_ENUM_SERIALIZATION(draft/persqueue_common.pb.h)
- GENERATE_ENUM_SERIALIZATION(ydb_persqueue_cluster_discovery.pb.h)
+ GENERATE_ENUM_SERIALIZATION(ydb_persqueue_cluster_discovery.pb.h)
GENERATE_ENUM_SERIALIZATION(draft/datastreams.pb.h)
ENDIF()
diff --git a/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto b/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto
index f3176fb39d..f39c900943 100644
--- a/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto
+++ b/ydb/public/api/protos/ydb_persqueue_cluster_discovery.proto
@@ -1,86 +1,86 @@
-syntax = "proto3";
-
-import "google/protobuf/empty.proto";
-
+syntax = "proto3";
+
+import "google/protobuf/empty.proto";
+
import "ydb/public/api/protos/ydb_operation.proto";
-
-package Ydb.PersQueue.ClusterDiscovery;
-
-option java_package = "com.yandex.ydb.persqueue.cluster_discovery";
-option cc_enable_arenas = true;
-
-
-message WriteSessionParams {
- // Path to the topic to write to.
- string topic = 1;
+
+package Ydb.PersQueue.ClusterDiscovery;
+
+option java_package = "com.yandex.ydb.persqueue.cluster_discovery";
+option cc_enable_arenas = true;
+
+
+message WriteSessionParams {
+ // Path to the topic to write to.
+ string topic = 1;
// Message group identifier.
- bytes source_id = 2;
- // Partition group to write to. 0 by default.
- uint32 partition_group = 3;
- // Force the specified cluster via its name. Leave it empty by default.
- string preferred_cluster_name = 4;
-}
-
-message ClusterInfo {
- // A host discovery endpoint to use at the next step.
- string endpoint = 1;
- // An official cluster name.
- string name = 2;
- // Is the cluster available right now?
- bool available = 3;
-};
-
-message ReadSessionParams {
- // Path to the topic to read from.
- string topic = 1;
- // Read mode is set according to the read rule.
- oneof read_rule {
- string mirror_to_cluster = 2;
- google.protobuf.Empty all_original = 3;
- }
-}
-
-message WriteSessionClusters {
- // Ordered clusters with statuses.
- repeated ClusterInfo clusters = 1;
-
- enum SelectionReason {
- SELECTION_REASON_UNSPECIFIED = 0;
- CLIENT_PREFERENCE = 1;
- CLIENT_LOCATION = 2;
- CONSISTENT_DISTRIBUTION = 3;
- }
-
- // The reason why a particular cluster was prioritized.
- SelectionReason primary_cluster_selection_reason = 2;
-}
-
-message ReadSessionClusters {
- // Ordered clusters with statuses.
- repeated ClusterInfo clusters = 1;
-}
-
-message DiscoverClustersRequest {
- Ydb.Operations.OperationParams operation_params = 1;
-
- // Clusters will be discovered separately for each element of the list.
- repeated WriteSessionParams write_sessions = 2;
- repeated ReadSessionParams read_sessions = 3;
-
- // Latest clusters status version known to the client application. Use 0 by default.
- int64 minimal_version = 4;
-}
-
-message DiscoverClustersResponse {
- // Operation contains the result of the request. Check the ydb_operation.proto.
- Ydb.Operations.Operation operation = 1;
-}
-
-message DiscoverClustersResult {
- // Discovered per-session clusters.
- repeated WriteSessionClusters write_sessions_clusters = 1;
- repeated ReadSessionClusters read_sessions_clusters = 2;
-
- // Latest clusters status version known to the cluster discovery service.
- int64 version = 3;
-}
+ bytes source_id = 2;
+ // Partition group to write to. 0 by default.
+ uint32 partition_group = 3;
+ // Force the specified cluster via its name. Leave it empty by default.
+ string preferred_cluster_name = 4;
+}
+
+message ClusterInfo {
+ // A host discovery endpoint to use at the next step.
+ string endpoint = 1;
+ // An official cluster name.
+ string name = 2;
+ // Is the cluster available right now?
+ bool available = 3;
+};
+
+message ReadSessionParams {
+ // Path to the topic to read from.
+ string topic = 1;
+ // Read mode is set according to the read rule.
+ oneof read_rule {
+ string mirror_to_cluster = 2;
+ google.protobuf.Empty all_original = 3;
+ }
+}
+
+message WriteSessionClusters {
+ // Ordered clusters with statuses.
+ repeated ClusterInfo clusters = 1;
+
+ enum SelectionReason {
+ SELECTION_REASON_UNSPECIFIED = 0;
+ CLIENT_PREFERENCE = 1;
+ CLIENT_LOCATION = 2;
+ CONSISTENT_DISTRIBUTION = 3;
+ }
+
+ // The reason why a particular cluster was prioritized.
+ SelectionReason primary_cluster_selection_reason = 2;
+}
+
+message ReadSessionClusters {
+ // Ordered clusters with statuses.
+ repeated ClusterInfo clusters = 1;
+}
+
+message DiscoverClustersRequest {
+ Ydb.Operations.OperationParams operation_params = 1;
+
+ // Clusters will be discovered separately for each element of the list.
+ repeated WriteSessionParams write_sessions = 2;
+ repeated ReadSessionParams read_sessions = 3;
+
+ // Latest clusters status version known to the client application. Use 0 by default.
+ int64 minimal_version = 4;
+}
+
+message DiscoverClustersResponse {
+ // Operation contains the result of the request. Check the ydb_operation.proto.
+ Ydb.Operations.Operation operation = 1;
+}
+
+message DiscoverClustersResult {
+ // Discovered per-session clusters.
+ repeated WriteSessionClusters write_sessions_clusters = 1;
+ repeated ReadSessionClusters read_sessions_clusters = 2;
+
+ // Latest clusters status version known to the cluster discovery service.
+ int64 version = 3;
+}
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp
index 5167c2c76d..aabd11d4ca 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp
+++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.cpp
@@ -1,81 +1,81 @@
-#include "cluster_discovery_service.h"
-#include "cluster_discovery_worker.h"
-
-#include "counters.h"
-
+#include "cluster_discovery_service.h"
+#include "cluster_discovery_worker.h"
+
+#include "counters.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/grpc_services/grpc_request_proxy.h>
#include <ydb/core/mind/address_classification/net_classifier.h>
#include <ydb/core/mon/mon.h>
#include <ydb/core/persqueue/cluster_tracker.h>
#include <ydb/core/util/address_classifier.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
-
-#include <algorithm>
-
-
-namespace NKikimr::NPQ::NClusterDiscovery {
-
-using namespace NCounters;
-
-inline auto& Ctx() {
- return TActivationContext::AsActorContext();
-}
-
-class TClusterDiscoveryServiceActor: public TActorBootstrapped<TClusterDiscoveryServiceActor> {
-public:
- TClusterDiscoveryServiceActor(TIntrusivePtr<NMonitoring::TDynamicCounters> counters)
- : RawCounters(counters)
- , Counters(BuildCounters())
- {
- }
-
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_DISCOVERY;
- }
-
- const auto& Cfg() const {
- return AppData(Ctx())->PQClusterDiscoveryConfig;
- }
-
- void Bootstrap() {
- Become(&TThis::Initing);
-
- SubscribeToNetClassifier();
- SubscribeToClusterTracker();
-
- InitCloudNetData();
-
- RegisterMonitoringPage();
-
- StartPeriodicalMonitoring();
- }
-
-private:
- void InitCloudNetData() {
- if (Cfg().HasCloudNetData()) {
- CloudNetworksClassifier = NAddressClassifier::BuildLabeledAddressClassifierFromNetData(Cfg().GetCloudNetData());
-
- Y_VERIFY(CloudNetworksClassifier); // the config is expected to be correct if specified
- }
- }
-
- void SubscribeToNetClassifier() {
- Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
- }
-
- void SubscribeToClusterTracker() {
- Send(NPQ::NClusterTracker::MakeClusterTrackerID(), new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe);
- }
-
- static TString FormatTs(const TMaybe<TInstant>& time) {
- if (time) {
- return time->ToRfc822String();
- }
- return "NULL";
- }
-
+
+#include <algorithm>
+
+
+namespace NKikimr::NPQ::NClusterDiscovery {
+
+using namespace NCounters;
+
+inline auto& Ctx() {
+ return TActivationContext::AsActorContext();
+}
+
+class TClusterDiscoveryServiceActor: public TActorBootstrapped<TClusterDiscoveryServiceActor> {
+public:
+ TClusterDiscoveryServiceActor(TIntrusivePtr<NMonitoring::TDynamicCounters> counters)
+ : RawCounters(counters)
+ , Counters(BuildCounters())
+ {
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::PERSQUEUE_CLUSTER_DISCOVERY;
+ }
+
+ const auto& Cfg() const {
+ return AppData(Ctx())->PQClusterDiscoveryConfig;
+ }
+
+ void Bootstrap() {
+ Become(&TThis::Initing);
+
+ SubscribeToNetClassifier();
+ SubscribeToClusterTracker();
+
+ InitCloudNetData();
+
+ RegisterMonitoringPage();
+
+ StartPeriodicalMonitoring();
+ }
+
+private:
+ void InitCloudNetData() {
+ if (Cfg().HasCloudNetData()) {
+ CloudNetworksClassifier = NAddressClassifier::BuildLabeledAddressClassifierFromNetData(Cfg().GetCloudNetData());
+
+ Y_VERIFY(CloudNetworksClassifier); // the config is expected to be correct if specified
+ }
+ }
+
+ void SubscribeToNetClassifier() {
+ Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
+ }
+
+ void SubscribeToClusterTracker() {
+ Send(NPQ::NClusterTracker::MakeClusterTrackerID(), new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe);
+ }
+
+ static TString FormatTs(const TMaybe<TInstant>& time) {
+ if (time) {
+ return time->ToRfc822String();
+ }
+ return "NULL";
+ }
+
bool IsHealthy(bool useLocalEnabled = false) const {
bool isLocalEnabled = false;
if (ClustersList) {
@@ -86,216 +86,216 @@ private:
}
return ClustersList && DatacenterClassifier && (isLocalEnabled || !useLocalEnabled);
- }
-
- TString MakeReport() const {
- TStringBuilder message;
- message << "PersQueue Cluster Discovery Service info: \n";
- message << "\tOverall status: " << (IsHealthy() ? "GOOD" : "BAD") << "\n\n";
-
- message << "\tInitial NetClassifier response ts: " << FormatTs(InitialNetClassifierResponseTs) << "\n";
- message << "\tInitial ClusterTracker response ts: " << FormatTs(InitialClusterTrackerResponseTs) << "\n";
- message << "\tFully initialized at: " << FormatTs(FullyInitializedTs) << "\n\n";
-
- message << "\tNetClassifier: " << (DatacenterClassifier ? "OK" : "NULL") << "\n";
- message << "\tNetClassifier NetData update ts: " << FormatTs(NetDataUpdateTs) << "\n\n";
-
- message << "\tClusters list: " << (ClustersList ? "OK" : "NULL") << "\n";
- message << "\tClusters list update ts: " << FormatTs(ClustersListUpdateTs) << "\n";
- if (ClustersList && ClustersList->Clusters.size()) {
- message << "\tClusters " << "(" << ClustersList->Clusters.size() << "): " << "\n";
- for (const auto& cluster : ClustersList->Clusters) {
- message << "\t\t" << cluster.DebugString() << "\n";
- }
- }
-
- message << "\n";
- message << "\tNetClassifier updates count: " << NetClassifierUpdatesCount << "\n";
- message << "\tClusters list updates count: " << ClustersListUpdatesCount << "\n";
-
- return TString(message);
- }
-
- void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) {
- const TStringBuf path = ev->Get()->Request.GetPath();
-
- TStringBuilder responseContent;
- if (path.EndsWith("/health")) {
+ }
+
+ TString MakeReport() const {
+ TStringBuilder message;
+ message << "PersQueue Cluster Discovery Service info: \n";
+ message << "\tOverall status: " << (IsHealthy() ? "GOOD" : "BAD") << "\n\n";
+
+ message << "\tInitial NetClassifier response ts: " << FormatTs(InitialNetClassifierResponseTs) << "\n";
+ message << "\tInitial ClusterTracker response ts: " << FormatTs(InitialClusterTrackerResponseTs) << "\n";
+ message << "\tFully initialized at: " << FormatTs(FullyInitializedTs) << "\n\n";
+
+ message << "\tNetClassifier: " << (DatacenterClassifier ? "OK" : "NULL") << "\n";
+ message << "\tNetClassifier NetData update ts: " << FormatTs(NetDataUpdateTs) << "\n\n";
+
+ message << "\tClusters list: " << (ClustersList ? "OK" : "NULL") << "\n";
+ message << "\tClusters list update ts: " << FormatTs(ClustersListUpdateTs) << "\n";
+ if (ClustersList && ClustersList->Clusters.size()) {
+ message << "\tClusters " << "(" << ClustersList->Clusters.size() << "): " << "\n";
+ for (const auto& cluster : ClustersList->Clusters) {
+ message << "\t\t" << cluster.DebugString() << "\n";
+ }
+ }
+
+ message << "\n";
+ message << "\tNetClassifier updates count: " << NetClassifierUpdatesCount << "\n";
+ message << "\tClusters list updates count: " << ClustersListUpdatesCount << "\n";
+
+ return TString(message);
+ }
+
+ void HandleHttpRequest(NMon::TEvHttpInfo::TPtr& ev) {
+ const TStringBuf path = ev->Get()->Request.GetPath();
+
+ TStringBuilder responseContent;
+ if (path.EndsWith("/health")) {
static const char HTTPNOTAVAIL_H[] = "HTTP/1.1 418 I'm a teapot\r\nConnection: Close\r\n\r\nDiscovery service is disabled on the node\r\n";
responseContent << (IsHealthy() ? NMonitoring::HTTPOKTEXT : HTTPNOTAVAIL_H) << "Service statuses: 200 - OK, 418 - DISABLED";
} else if (path.EndsWith("/ping")) {
static const char HTTPNOTAVAIL_P[] = "HTTP/1.1 418 I'm a teapot\r\nConnection: Close\r\n\r\nDiscovery service is disabled on the node and local cluster is disabled\r\n";
responseContent << (IsHealthy(true) ? NMonitoring::HTTPOKTEXT : HTTPNOTAVAIL_P) << "Service statuses: 200 - OK, 418 - DISABLED";
} else{
- responseContent << NMonitoring::HTTPOKTEXT << MakeReport();
- }
-
- Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(responseContent), 0, NMon::IEvHttpInfoRes::EContentType::Custom));
- }
-
- void UpdateNetClassifier(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
- DatacenterClassifier = ev->Get()->Classifier;
- NetDataUpdateTs = ev->Get()->NetDataUpdateTimestamp;
-
- Counters = BuildCounters();
-
- ++NetClassifierUpdatesCount;
-
- UpdateHealthProbe();
- }
-
- void HandleClassifierUpdateWhileIniting(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
- UpdateNetClassifier(ev);
-
- InitialNetClassifierResponseTs = Ctx().Now();
-
- CompleteInitialization();
- }
-
- void UpdateClustersList(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
- ClustersListUpdateTs = ev->Get()->ClustersListUpdateTimestamp;
-
- if (!ClustersList || !(*ClustersList == *ev->Get()->ClustersList)) {
- ClustersList = ev->Get()->ClustersList;
- Counters = BuildCounters();
- }
-
- ++ClustersListUpdatesCount;
-
- UpdateHealthProbe();
- }
-
- void HandleClustersUpdateWhileIniting(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
- UpdateClustersList(ev);
-
- InitialClusterTrackerResponseTs = Ctx().Now();
-
- CompleteInitialization();
- }
-
- void CompleteInitialization() {
- if (!InitialNetClassifierResponseTs || !InitialClusterTrackerResponseTs) {
- return;
- }
-
- FullyInitializedTs = Ctx().Now();
- Become(&TThis::Working);
- }
-
- void RegisterMonitoringPage() const {
- NActors::TMon* mon = AppData(Ctx())->Mon;
- if (mon) {
- NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors");
- mon->RegisterActorPage(page, "pqcd", "PersQueue Cluster Discovery", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID);
- }
- }
-
- STATEFN(Initing) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileIniting);
- hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileIniting);
- hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileIniting);
- hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
- hFunc(TEvents::TEvWakeup, UpdateTimedCounters);
- }
- }
-
- void RespondServiceUnavailable(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) {
- Counters->DroppedRequestsCount->Inc();
-
+ responseContent << NMonitoring::HTTPOKTEXT << MakeReport();
+ }
+
+ Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(responseContent), 0, NMon::IEvHttpInfoRes::EContentType::Custom));
+ }
+
+ void UpdateNetClassifier(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
+ DatacenterClassifier = ev->Get()->Classifier;
+ NetDataUpdateTs = ev->Get()->NetDataUpdateTimestamp;
+
+ Counters = BuildCounters();
+
+ ++NetClassifierUpdatesCount;
+
+ UpdateHealthProbe();
+ }
+
+ void HandleClassifierUpdateWhileIniting(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
+ UpdateNetClassifier(ev);
+
+ InitialNetClassifierResponseTs = Ctx().Now();
+
+ CompleteInitialization();
+ }
+
+ void UpdateClustersList(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
+ ClustersListUpdateTs = ev->Get()->ClustersListUpdateTimestamp;
+
+ if (!ClustersList || !(*ClustersList == *ev->Get()->ClustersList)) {
+ ClustersList = ev->Get()->ClustersList;
+ Counters = BuildCounters();
+ }
+
+ ++ClustersListUpdatesCount;
+
+ UpdateHealthProbe();
+ }
+
+ void HandleClustersUpdateWhileIniting(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
+ UpdateClustersList(ev);
+
+ InitialClusterTrackerResponseTs = Ctx().Now();
+
+ CompleteInitialization();
+ }
+
+ void CompleteInitialization() {
+ if (!InitialNetClassifierResponseTs || !InitialClusterTrackerResponseTs) {
+ return;
+ }
+
+ FullyInitializedTs = Ctx().Now();
+ Become(&TThis::Working);
+ }
+
+ void RegisterMonitoringPage() const {
+ NActors::TMon* mon = AppData(Ctx())->Mon;
+ if (mon) {
+ NMonitoring::TIndexMonPage * page = mon->RegisterIndexPage("actors", "Actors");
+ mon->RegisterActorPage(page, "pqcd", "PersQueue Cluster Discovery", false, Ctx().ExecutorThread.ActorSystem, Ctx().SelfID);
+ }
+ }
+
+ STATEFN(Initing) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileIniting);
+ hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileIniting);
+ hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileIniting);
+ hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
+ hFunc(TEvents::TEvWakeup, UpdateTimedCounters);
+ }
+ }
+
+ void RespondServiceUnavailable(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) {
+ Counters->DroppedRequestsCount->Inc();
+
ev->Get()->ReplyWithYdbStatus(Ydb::StatusIds::UNAVAILABLE);
- }
-
- void HandleDiscoverPQClustersRequestWhileIniting(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) {
- Counters->TotalRequestsCount->Inc();
-
- RespondServiceUnavailable(ev);
- }
-
- void HandleClassifierUpdateWhileWorking(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
- UpdateNetClassifier(ev);
- }
-
- void HandleClustersUpdateWhileWorking(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
- UpdateClustersList(ev);
- }
-
- void HandleDiscoverPQClustersRequestWhileWorking(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) {
- Counters->TotalRequestsCount->Inc();
-
- if (!IsHealthy()) {
- RespondServiceUnavailable(ev);
- return;
- }
-
- IActor* actor = NWorker::CreateClusterDiscoveryWorker(ev, DatacenterClassifier, CloudNetworksClassifier, ClustersList, Counters);
- Register(actor, TMailboxType::HTSwap, AppData(Ctx())->UserPoolId);
- }
-
- STATEFN(Working) {
- switch (ev->GetTypeRewrite()) {
- hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileWorking);
- hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileWorking);
- hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileWorking);
- hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
- hFunc(TEvents::TEvWakeup, UpdateTimedCounters);
- }
- }
-
- // Monitoring features
-
- void StartPeriodicalMonitoring() {
- Y_VERIFY(Cfg().GetTimedCountersUpdateIntervalSeconds());
-
- Send(Ctx().SelfID, new TEvents::TEvWakeup);
- }
-
- void UpdateHealthProbe() {
- (*Counters->HealthProbe) = IsHealthy();
- }
-
- void UpdateTimedCounters(TEvents::TEvWakeup::TPtr&) {
- if (NetDataUpdateTs) {
- (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTs->Seconds();
- }
-
- if (ClustersListUpdateTs) {
- (*Counters->ClustersListUpdateLagSeconds) = Ctx().Now().Seconds() - ClustersListUpdateTs->Seconds();
- }
-
- UpdateHealthProbe();
-
- Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvents::TEvWakeup);
- }
-
- TClusterDiscoveryCounters::TPtr BuildCounters() {
- return MakeIntrusive<TClusterDiscoveryCounters>(RawCounters, ClustersList, DatacenterClassifier);
- }
-
-private:
+ }
+
+ void HandleDiscoverPQClustersRequestWhileIniting(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) {
+ Counters->TotalRequestsCount->Inc();
+
+ RespondServiceUnavailable(ev);
+ }
+
+ void HandleClassifierUpdateWhileWorking(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev) {
+ UpdateNetClassifier(ev);
+ }
+
+ void HandleClustersUpdateWhileWorking(NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
+ UpdateClustersList(ev);
+ }
+
+ void HandleDiscoverPQClustersRequestWhileWorking(NGRpcService::TEvDiscoverPQClustersRequest::TPtr& ev) {
+ Counters->TotalRequestsCount->Inc();
+
+ if (!IsHealthy()) {
+ RespondServiceUnavailable(ev);
+ return;
+ }
+
+ IActor* actor = NWorker::CreateClusterDiscoveryWorker(ev, DatacenterClassifier, CloudNetworksClassifier, ClustersList, Counters);
+ Register(actor, TMailboxType::HTSwap, AppData(Ctx())->UserPoolId);
+ }
+
+ STATEFN(Working) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, HandleClassifierUpdateWhileWorking);
+ hFunc(NClusterTracker::TEvClusterTracker::TEvClustersUpdate, HandleClustersUpdateWhileWorking);
+ hFunc(NGRpcService::TEvDiscoverPQClustersRequest, HandleDiscoverPQClustersRequestWhileWorking);
+ hFunc(NMon::TEvHttpInfo, HandleHttpRequest);
+ hFunc(TEvents::TEvWakeup, UpdateTimedCounters);
+ }
+ }
+
+ // Monitoring features
+
+ void StartPeriodicalMonitoring() {
+ Y_VERIFY(Cfg().GetTimedCountersUpdateIntervalSeconds());
+
+ Send(Ctx().SelfID, new TEvents::TEvWakeup);
+ }
+
+ void UpdateHealthProbe() {
+ (*Counters->HealthProbe) = IsHealthy();
+ }
+
+ void UpdateTimedCounters(TEvents::TEvWakeup::TPtr&) {
+ if (NetDataUpdateTs) {
+ (*Counters->NetDataUpdateLagSeconds) = Ctx().Now().Seconds() - NetDataUpdateTs->Seconds();
+ }
+
+ if (ClustersListUpdateTs) {
+ (*Counters->ClustersListUpdateLagSeconds) = Ctx().Now().Seconds() - ClustersListUpdateTs->Seconds();
+ }
+
+ UpdateHealthProbe();
+
+ Schedule(TDuration::Seconds(Cfg().GetTimedCountersUpdateIntervalSeconds()), new TEvents::TEvWakeup);
+ }
+
+ TClusterDiscoveryCounters::TPtr BuildCounters() {
+ return MakeIntrusive<TClusterDiscoveryCounters>(RawCounters, ClustersList, DatacenterClassifier);
+ }
+
+private:
TActorId NetClassifierActorId;
TActorId ClusterTrackerActorId;
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier;
- NAddressClassifier::TLabeledAddressClassifier::TConstPtr CloudNetworksClassifier;
- TMaybe<TInstant> NetDataUpdateTs;
-
- NClusterTracker::TClustersList::TConstPtr ClustersList;
- TMaybe<TInstant> ClustersListUpdateTs;
-
- TMaybe<TInstant> InitialNetClassifierResponseTs;
- TMaybe<TInstant> InitialClusterTrackerResponseTs;
-
- TMaybe<TInstant> FullyInitializedTs;
-
- size_t NetClassifierUpdatesCount = 0;
- size_t ClustersListUpdatesCount = 0;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> RawCounters;
- TClusterDiscoveryCounters::TPtr Counters;
-};
-
-NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) {
- return new TClusterDiscoveryServiceActor(counters);
-}
-
-} // namespace NKikimr::NPQ::NClusterDiscovery
+ NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier;
+ NAddressClassifier::TLabeledAddressClassifier::TConstPtr CloudNetworksClassifier;
+ TMaybe<TInstant> NetDataUpdateTs;
+
+ NClusterTracker::TClustersList::TConstPtr ClustersList;
+ TMaybe<TInstant> ClustersListUpdateTs;
+
+ TMaybe<TInstant> InitialNetClassifierResponseTs;
+ TMaybe<TInstant> InitialClusterTrackerResponseTs;
+
+ TMaybe<TInstant> FullyInitializedTs;
+
+ size_t NetClassifierUpdatesCount = 0;
+ size_t ClustersListUpdatesCount = 0;
+
+ TIntrusivePtr<NMonitoring::TDynamicCounters> RawCounters;
+ TClusterDiscoveryCounters::TPtr Counters;
+};
+
+NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters) {
+ return new TClusterDiscoveryServiceActor(counters);
+}
+
+} // namespace NKikimr::NPQ::NClusterDiscovery
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h
index 65f91409c4..fec5708c15 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h
+++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service.h
@@ -1,22 +1,22 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/base/events.h>
-
+
#include <library/cpp/actors/core/actor.h>
#include <library/cpp/actors/core/actorid.h>
#include <library/cpp/actors/core/defs.h>
#include <library/cpp/actors/core/event_local.h>
-
+
#include <library/cpp/monlib/dynamic_counters/counters.h>
-
-namespace NKikimr::NPQ::NClusterDiscovery {
+
+namespace NKikimr::NPQ::NClusterDiscovery {
using NActors::TActorId;
-
+
inline TActorId MakeClusterDiscoveryServiceID() {
const char x[TActorId::MaxServiceIDLength] = "pq_discosvc";
return TActorId(0, TStringBuf(x, TActorId::MaxServiceIDLength));
- }
-
- NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters);
-
-} // namespace NKikimr::NPQ::NClusterDiscovery
+ }
+
+ NActors::IActor* CreateClusterDiscoveryService(TIntrusivePtr<NMonitoring::TDynamicCounters> counters);
+
+} // namespace NKikimr::NPQ::NClusterDiscovery
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp
index 2d1612ea1e..34721e262d 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp
+++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_service_ut.cpp
@@ -1,684 +1,684 @@
#include <ydb/services/persqueue_cluster_discovery/counters.h>
-
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/counters.h>
-
+
#include <ydb/core/mind/address_classification/net_classifier.h>
#include <ydb/core/persqueue/cluster_tracker.h>
#include <ydb/core/testlib/test_pq_client.h>
-
+
#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h>
-
+
#include <library/cpp/actors/http/http_proxy.h>
-
+
#include <library/cpp/testing/unittest/tests_data.h>
#include <library/cpp/testing/unittest/registar.h>
#include <google/protobuf/text_format.h>
-
-#include <grpc++/client_context.h>
-#include <grpc++/create_channel.h>
-
-#include <util/stream/file.h>
-#include <util/system/tempfile.h>
-
-namespace NKikimr::NPQCDTests {
-
-using namespace Tests;
-using namespace NKikimrClient;
-using namespace Ydb::PersQueue::ClusterDiscovery;
-
-struct TFancyGRpcWrapper {
- TFancyGRpcWrapper(const ui16 grpcPort) {
- Channel_ = grpc::CreateChannel("localhost:" + ToString(grpcPort), grpc::InsecureChannelCredentials());
- Stub_ = Ydb::PersQueue::V1::ClusterDiscoveryService::NewStub(Channel_);
- }
-
- auto PerformRpc(const DiscoverClustersRequest& request, DiscoverClustersResult& result) {
- grpc::ClientContext context;
- DiscoverClustersResponse response;
-
- const auto status = Stub_->DiscoverClusters(&context, request, &response);
-
- UNIT_ASSERT(status.ok());
- UNIT_ASSERT(response.operation().ready());
-
- const auto opStatus = response.operation().status();
-
- if (opStatus == Ydb::StatusIds::SUCCESS) {
- response.operation().result().UnpackTo(&result);
- }
-
- return opStatus;
- }
-
- void Wait() const {
- Sleep(TDuration::MilliSeconds(100));
- }
-
- void EnsureServiceUnavailability() {
- DiscoverClustersRequest request;
- DiscoverClustersResult result;
-
- UNIT_ASSERT_VALUES_EQUAL(PerformRpc(request, result), Ydb::StatusIds::UNAVAILABLE);
- }
-
- void WaitForExactClustersDataVersion(const i64 requiredVersion) {
- while (true) {
- DiscoverClustersRequest request;
- DiscoverClustersResult result;
-
- if (PerformRpc(request, result) == Ydb::StatusIds::SUCCESS && result.version() == requiredVersion) {
- break;
- }
-
- UNIT_ASSERT(result.version() < requiredVersion);
-
- Wait();
- }
- }
-
-private:
- std::shared_ptr<grpc::Channel> Channel_;
- std::unique_ptr<Ydb::PersQueue::V1::ClusterDiscoveryService::Stub> Stub_;
-};
-
-static void VerifyClusterInfo(const ClusterInfo& clusterInfo, const TString& name, const TString& endpoint, bool isAvailable) {
- UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.name(), name);
- UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.endpoint(), endpoint);
- UNIT_ASSERT_VALUES_EQUAL(clusterInfo.available(), isAvailable);
-}
-
-static void VerifyEqualClusterInfo(const ClusterInfo& c1, const ClusterInfo& c2) {
- VerifyClusterInfo(c1, c2.name(), c2.endpoint(), c2.available());
-}
-
-static void CallPQCDAndAssert(TFancyGRpcWrapper& wrapper,
- const DiscoverClustersRequest& request,
- DiscoverClustersResult& result,
- const i64 expectedVersion,
- const size_t expectedWriteSessionsClusters,
- const size_t expectedReadSessionsClusters)
-{
- UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::SUCCESS);
- UNIT_ASSERT_VALUES_EQUAL(result.version(), expectedVersion);
-
- UNIT_ASSERT_VALUES_EQUAL(result.write_sessions_clusters_size(), expectedWriteSessionsClusters);
- UNIT_ASSERT_VALUES_EQUAL(result.read_sessions_clusters_size(), expectedReadSessionsClusters);
-}
-
-static void CheckReadDiscoveryHandlers(TFancyGRpcWrapper& wrapper,
- const ClusterInfo& c1Info,
- const ClusterInfo& c2Info,
- i64 expectedVersion)
-{
- {
- // EMPTY
- DiscoverClustersRequest request;
- DiscoverClustersResult result;
-
- CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 0);
- }
- {
- // READ ALL ORIGINAL
- DiscoverClustersRequest request;
- auto* readSessionParams = request.add_read_sessions();
- readSessionParams->mutable_all_original();
-
- DiscoverClustersResult result;
-
- CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1);
-
- const auto& readSessionClusters = result.read_sessions_clusters(0);
-
- UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 2);
-
- VerifyEqualClusterInfo(readSessionClusters.clusters(0), c1Info);
- VerifyEqualClusterInfo(readSessionClusters.clusters(1), c2Info);
- }
- {
- // READ MIRRORED
- DiscoverClustersRequest request;
- auto* readSessionParams = request.add_read_sessions();
- readSessionParams->set_mirror_to_cluster("dc2");
- readSessionParams->set_topic("megatopic");
-
- DiscoverClustersResult result;
-
- CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1);
-
- const auto& readSessionClusters = result.read_sessions_clusters(0);
-
- UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 1);
-
- VerifyEqualClusterInfo(readSessionClusters.clusters(0), c2Info);
- }
- {
- // READ MIRRORED INEXISTING
- DiscoverClustersRequest request;
- auto* readSessionParams = request.add_read_sessions();
- readSessionParams->set_mirror_to_cluster("trololo");
-
- DiscoverClustersResult result;
-
- UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST);
- }
-}
-
-static void CheckWriteDiscoveryHandlers(TFancyGRpcWrapper& wrapper,
- const ClusterInfo& c1Info,
- const ClusterInfo& c2Info,
- i64 expectedVersion)
-{
- {
- // WRITE SIMPLE
- DiscoverClustersRequest request;
- {
- // causes the clusters to be arranged in the following order: #1, #2
- auto* writeSessionParams = request.add_write_sessions();
- writeSessionParams->set_topic("topic12");
- writeSessionParams->set_source_id("topic1.log");
- writeSessionParams->set_partition_group(42);
- }
- {
- // causes the clusters to be arranged in the following order: #2, #1
- auto* writeSessionParams = request.add_write_sessions();
- writeSessionParams->set_topic("topic1");
- writeSessionParams->set_source_id("topic1.log");
- writeSessionParams->set_partition_group(42);
- }
- {
- // preferred cluster option causes the clusters to be arranged in the following order: #2, #1
- auto* writeSessionParams = request.add_write_sessions();
- writeSessionParams->set_topic("topic12");
- writeSessionParams->set_source_id("topic1.log");
- writeSessionParams->set_partition_group(42);
- writeSessionParams->set_preferred_cluster_name("dc2");
- }
-
- DiscoverClustersResult result;
-
- CallPQCDAndAssert(wrapper, request, result, expectedVersion, 3, 0);
-
- {
- const auto& writeSessionClusters = result.write_sessions_clusters(0);
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
-
- VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c1Info);
- VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c2Info);
-
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION);
- }
- {
- const auto& writeSessionClusters = result.write_sessions_clusters(1);
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
-
- VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info);
- VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info);
-
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION);
- }
- {
- const auto& writeSessionClusters = result.write_sessions_clusters(2);
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
-
- VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info);
- VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info);
-
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_PREFERENCE);
- }
- }
- {
- // WRITE INEXISTENT PREFERRED CLUSTER
- DiscoverClustersRequest request;
- {
- auto* writeSessionParams = request.add_write_sessions();
- writeSessionParams->set_topic("topic");
- writeSessionParams->set_source_id("topic1.log");
- writeSessionParams->set_preferred_cluster_name("dc777");
- }
-
- DiscoverClustersResult result;
-
- UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST);
- }
-}
-
-class TPQCDServer {
-public:
- TPQCDServer(bool enablePQCD = true)
- : PortManager_()
- , BusPort_(PortManager_.GetPort(2134))
- , GrpcPort_(PortManager_.GetPort(2135))
- , Settings_(NPersQueueTests::PQSettings(BusPort_, 1))
- {
- Settings_.PQConfig.SetClustersUpdateTimeoutSec(1);
-
- Settings_.PQClusterDiscoveryConfig.SetEnabled(enablePQCD);
- Settings_.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1);
- }
-
- ui16 BusPort() const {
- return BusPort_;
- }
-
- ui16 MonPort() const {
+
+#include <grpc++/client_context.h>
+#include <grpc++/create_channel.h>
+
+#include <util/stream/file.h>
+#include <util/system/tempfile.h>
+
+namespace NKikimr::NPQCDTests {
+
+using namespace Tests;
+using namespace NKikimrClient;
+using namespace Ydb::PersQueue::ClusterDiscovery;
+
+struct TFancyGRpcWrapper {
+ TFancyGRpcWrapper(const ui16 grpcPort) {
+ Channel_ = grpc::CreateChannel("localhost:" + ToString(grpcPort), grpc::InsecureChannelCredentials());
+ Stub_ = Ydb::PersQueue::V1::ClusterDiscoveryService::NewStub(Channel_);
+ }
+
+ auto PerformRpc(const DiscoverClustersRequest& request, DiscoverClustersResult& result) {
+ grpc::ClientContext context;
+ DiscoverClustersResponse response;
+
+ const auto status = Stub_->DiscoverClusters(&context, request, &response);
+
+ UNIT_ASSERT(status.ok());
+ UNIT_ASSERT(response.operation().ready());
+
+ const auto opStatus = response.operation().status();
+
+ if (opStatus == Ydb::StatusIds::SUCCESS) {
+ response.operation().result().UnpackTo(&result);
+ }
+
+ return opStatus;
+ }
+
+ void Wait() const {
+ Sleep(TDuration::MilliSeconds(100));
+ }
+
+ void EnsureServiceUnavailability() {
+ DiscoverClustersRequest request;
+ DiscoverClustersResult result;
+
+ UNIT_ASSERT_VALUES_EQUAL(PerformRpc(request, result), Ydb::StatusIds::UNAVAILABLE);
+ }
+
+ void WaitForExactClustersDataVersion(const i64 requiredVersion) {
+ while (true) {
+ DiscoverClustersRequest request;
+ DiscoverClustersResult result;
+
+ if (PerformRpc(request, result) == Ydb::StatusIds::SUCCESS && result.version() == requiredVersion) {
+ break;
+ }
+
+ UNIT_ASSERT(result.version() < requiredVersion);
+
+ Wait();
+ }
+ }
+
+private:
+ std::shared_ptr<grpc::Channel> Channel_;
+ std::unique_ptr<Ydb::PersQueue::V1::ClusterDiscoveryService::Stub> Stub_;
+};
+
+static void VerifyClusterInfo(const ClusterInfo& clusterInfo, const TString& name, const TString& endpoint, bool isAvailable) {
+ UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.name(), name);
+ UNIT_ASSERT_STRINGS_EQUAL(clusterInfo.endpoint(), endpoint);
+ UNIT_ASSERT_VALUES_EQUAL(clusterInfo.available(), isAvailable);
+}
+
+static void VerifyEqualClusterInfo(const ClusterInfo& c1, const ClusterInfo& c2) {
+ VerifyClusterInfo(c1, c2.name(), c2.endpoint(), c2.available());
+}
+
+static void CallPQCDAndAssert(TFancyGRpcWrapper& wrapper,
+ const DiscoverClustersRequest& request,
+ DiscoverClustersResult& result,
+ const i64 expectedVersion,
+ const size_t expectedWriteSessionsClusters,
+ const size_t expectedReadSessionsClusters)
+{
+ UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::SUCCESS);
+ UNIT_ASSERT_VALUES_EQUAL(result.version(), expectedVersion);
+
+ UNIT_ASSERT_VALUES_EQUAL(result.write_sessions_clusters_size(), expectedWriteSessionsClusters);
+ UNIT_ASSERT_VALUES_EQUAL(result.read_sessions_clusters_size(), expectedReadSessionsClusters);
+}
+
+static void CheckReadDiscoveryHandlers(TFancyGRpcWrapper& wrapper,
+ const ClusterInfo& c1Info,
+ const ClusterInfo& c2Info,
+ i64 expectedVersion)
+{
+ {
+ // EMPTY
+ DiscoverClustersRequest request;
+ DiscoverClustersResult result;
+
+ CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 0);
+ }
+ {
+ // READ ALL ORIGINAL
+ DiscoverClustersRequest request;
+ auto* readSessionParams = request.add_read_sessions();
+ readSessionParams->mutable_all_original();
+
+ DiscoverClustersResult result;
+
+ CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1);
+
+ const auto& readSessionClusters = result.read_sessions_clusters(0);
+
+ UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 2);
+
+ VerifyEqualClusterInfo(readSessionClusters.clusters(0), c1Info);
+ VerifyEqualClusterInfo(readSessionClusters.clusters(1), c2Info);
+ }
+ {
+ // READ MIRRORED
+ DiscoverClustersRequest request;
+ auto* readSessionParams = request.add_read_sessions();
+ readSessionParams->set_mirror_to_cluster("dc2");
+ readSessionParams->set_topic("megatopic");
+
+ DiscoverClustersResult result;
+
+ CallPQCDAndAssert(wrapper, request, result, expectedVersion, 0, 1);
+
+ const auto& readSessionClusters = result.read_sessions_clusters(0);
+
+ UNIT_ASSERT_VALUES_EQUAL(readSessionClusters.clusters_size(), 1);
+
+ VerifyEqualClusterInfo(readSessionClusters.clusters(0), c2Info);
+ }
+ {
+ // READ MIRRORED INEXISTING
+ DiscoverClustersRequest request;
+ auto* readSessionParams = request.add_read_sessions();
+ readSessionParams->set_mirror_to_cluster("trololo");
+
+ DiscoverClustersResult result;
+
+ UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST);
+ }
+}
+
+static void CheckWriteDiscoveryHandlers(TFancyGRpcWrapper& wrapper,
+ const ClusterInfo& c1Info,
+ const ClusterInfo& c2Info,
+ i64 expectedVersion)
+{
+ {
+ // WRITE SIMPLE
+ DiscoverClustersRequest request;
+ {
+ // causes the clusters to be arranged in the following order: #1, #2
+ auto* writeSessionParams = request.add_write_sessions();
+ writeSessionParams->set_topic("topic12");
+ writeSessionParams->set_source_id("topic1.log");
+ writeSessionParams->set_partition_group(42);
+ }
+ {
+ // causes the clusters to be arranged in the following order: #2, #1
+ auto* writeSessionParams = request.add_write_sessions();
+ writeSessionParams->set_topic("topic1");
+ writeSessionParams->set_source_id("topic1.log");
+ writeSessionParams->set_partition_group(42);
+ }
+ {
+ // preferred cluster option causes the clusters to be arranged in the following order: #2, #1
+ auto* writeSessionParams = request.add_write_sessions();
+ writeSessionParams->set_topic("topic12");
+ writeSessionParams->set_source_id("topic1.log");
+ writeSessionParams->set_partition_group(42);
+ writeSessionParams->set_preferred_cluster_name("dc2");
+ }
+
+ DiscoverClustersResult result;
+
+ CallPQCDAndAssert(wrapper, request, result, expectedVersion, 3, 0);
+
+ {
+ const auto& writeSessionClusters = result.write_sessions_clusters(0);
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
+
+ VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c1Info);
+ VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c2Info);
+
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION);
+ }
+ {
+ const auto& writeSessionClusters = result.write_sessions_clusters(1);
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
+
+ VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info);
+ VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info);
+
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION);
+ }
+ {
+ const auto& writeSessionClusters = result.write_sessions_clusters(2);
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
+
+ VerifyEqualClusterInfo(writeSessionClusters.clusters(0), c2Info);
+ VerifyEqualClusterInfo(writeSessionClusters.clusters(1), c1Info);
+
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_PREFERENCE);
+ }
+ }
+ {
+ // WRITE INEXISTENT PREFERRED CLUSTER
+ DiscoverClustersRequest request;
+ {
+ auto* writeSessionParams = request.add_write_sessions();
+ writeSessionParams->set_topic("topic");
+ writeSessionParams->set_source_id("topic1.log");
+ writeSessionParams->set_preferred_cluster_name("dc777");
+ }
+
+ DiscoverClustersResult result;
+
+ UNIT_ASSERT_VALUES_EQUAL(wrapper.PerformRpc(request, result), Ydb::StatusIds::BAD_REQUEST);
+ }
+}
+
+class TPQCDServer {
+public:
+ TPQCDServer(bool enablePQCD = true)
+ : PortManager_()
+ , BusPort_(PortManager_.GetPort(2134))
+ , GrpcPort_(PortManager_.GetPort(2135))
+ , Settings_(NPersQueueTests::PQSettings(BusPort_, 1))
+ {
+ Settings_.PQConfig.SetClustersUpdateTimeoutSec(1);
+
+ Settings_.PQClusterDiscoveryConfig.SetEnabled(enablePQCD);
+ Settings_.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1);
+ }
+
+ ui16 BusPort() const {
+ return BusPort_;
+ }
+
+ ui16 MonPort() const {
return Server_->GetRuntime()->GetMonPort();
- }
-
- ui16 GrpcPort() const {
- return GrpcPort_;
- }
-
- TServerSettings& MutableSettings() {
- return Settings_;
- }
-
- void SetNetDataViaFile(const TString& netDataTsv) {
- UNIT_ASSERT(!Server_);
-
- NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv");
- NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size());
- NetDataFile->FlushData();
-
- Settings_.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name());
- }
-
- void Run() {
- Server_ = MakeHolder<TServer>(Settings_);
+ }
+
+ ui16 GrpcPort() const {
+ return GrpcPort_;
+ }
+
+ TServerSettings& MutableSettings() {
+ return Settings_;
+ }
+
+ void SetNetDataViaFile(const TString& netDataTsv) {
+ UNIT_ASSERT(!Server_);
+
+ NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv");
+ NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size());
+ NetDataFile->FlushData();
+
+ Settings_.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name());
+ }
+
+ void Run() {
+ Server_ = MakeHolder<TServer>(Settings_);
Server_->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(GrpcPort_));
- }
-
- NPersQueueTests::TFlatMsgBusPQClient& PQClient() {
- UNIT_ASSERT(Server_);
-
- if (!PQClient_) {
- PQClient_ = MakeHolder<NPersQueueTests::TFlatMsgBusPQClient>(Settings_, GrpcPort_);
- }
-
- return *PQClient_;
- }
-
- auto& ActorSystem() {
- UNIT_ASSERT(Server_);
- UNIT_ASSERT(Server_->GetRuntime());
-
- return *Server_->GetRuntime();
- }
-
- NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr HttpRequest(const TString& path) {
- UNIT_ASSERT(Server_);
-
- if (!HttpProxyId_) {
- IActor* proxy = NHttp::CreateHttpProxy(Sensors_);
- HttpProxyId_ = ActorSystem().Register(proxy);
- }
-
+ }
+
+ NPersQueueTests::TFlatMsgBusPQClient& PQClient() {
+ UNIT_ASSERT(Server_);
+
+ if (!PQClient_) {
+ PQClient_ = MakeHolder<NPersQueueTests::TFlatMsgBusPQClient>(Settings_, GrpcPort_);
+ }
+
+ return *PQClient_;
+ }
+
+ auto& ActorSystem() {
+ UNIT_ASSERT(Server_);
+ UNIT_ASSERT(Server_->GetRuntime());
+
+ return *Server_->GetRuntime();
+ }
+
+ NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr HttpRequest(const TString& path) {
+ UNIT_ASSERT(Server_);
+
+ if (!HttpProxyId_) {
+ IActor* proxy = NHttp::CreateHttpProxy(Sensors_);
+ HttpProxyId_ = ActorSystem().Register(proxy);
+ }
+
TActorId clientId = ActorSystem().AllocateEdgeActor();
NHttp::THttpOutgoingRequestPtr httpRequest = NHttp::THttpOutgoingRequest::CreateRequestGet("http://[::1]:" + ToString(MonPort()) + path);
- ActorSystem().Send(new NActors::IEventHandle(HttpProxyId_, clientId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)), 0, true);
-
- return ActorSystem().GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingResponse>(clientId);
- }
-
- void Wait() {
- Sleep(TDuration::MilliSeconds(100));
- }
-
- auto ServiceCounters() const {
- UNIT_ASSERT(Server_);
- UNIT_ASSERT(Server_->GetGRpcServerRootCounters());
-
- return GetServiceCounters(Server_->GetGRpcServerRootCounters(), "persqueue")->GetSubgroup("subsystem", "cluster_discovery");
- }
-
- auto GetTimesResolvedByClassifier(const TString& datacenterLabel) const {
- return ServiceCounters()->GetSubgroup("resolved_dc", datacenterLabel)->GetCounter("TimesResolvedByClassifier", true)->GetAtomic();
- }
-
- bool CheckServiceHealth() {
- UNIT_ASSERT(Server_);
-
- while (true) {
- auto responsePtr = HttpRequest("/actors/pqcd/health");
- const auto& response = *responsePtr->Get();
-
- if (!response.Error && (response.Response->Status == "200") || (response.Response->Status == "418")) {
- UNIT_ASSERT(response.Response->Body);
- return response.Response->Status == "200";
- }
-
- Wait();
- }
-
- return false;
- }
-
- void WaitUntilHealthy() {
- while (true) {
- if (CheckServiceHealth()) {
- return;
- }
-
- Wait();
- }
- }
-
- void EnsureServiceUnavailability() {
- // check the response for 3 seconds
- for (size_t i = 0; i < 30; ++i) {
- UNIT_ASSERT(!CheckServiceHealth());
- Wait();
- }
-
- TFancyGRpcWrapper(GrpcPort_).EnsureServiceUnavailability();
- }
-
-private:
- TPortManager PortManager_;
- ui16 BusPort_;
- ui16 GrpcPort_;
- TServerSettings Settings_;
-
+ ActorSystem().Send(new NActors::IEventHandle(HttpProxyId_, clientId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(httpRequest)), 0, true);
+
+ return ActorSystem().GrabEdgeEvent<NHttp::TEvHttpProxy::TEvHttpIncomingResponse>(clientId);
+ }
+
+ void Wait() {
+ Sleep(TDuration::MilliSeconds(100));
+ }
+
+ auto ServiceCounters() const {
+ UNIT_ASSERT(Server_);
+ UNIT_ASSERT(Server_->GetGRpcServerRootCounters());
+
+ return GetServiceCounters(Server_->GetGRpcServerRootCounters(), "persqueue")->GetSubgroup("subsystem", "cluster_discovery");
+ }
+
+ auto GetTimesResolvedByClassifier(const TString& datacenterLabel) const {
+ return ServiceCounters()->GetSubgroup("resolved_dc", datacenterLabel)->GetCounter("TimesResolvedByClassifier", true)->GetAtomic();
+ }
+
+ bool CheckServiceHealth() {
+ UNIT_ASSERT(Server_);
+
+ while (true) {
+ auto responsePtr = HttpRequest("/actors/pqcd/health");
+ const auto& response = *responsePtr->Get();
+
+ if (!response.Error && (response.Response->Status == "200") || (response.Response->Status == "418")) {
+ UNIT_ASSERT(response.Response->Body);
+ return response.Response->Status == "200";
+ }
+
+ Wait();
+ }
+
+ return false;
+ }
+
+ void WaitUntilHealthy() {
+ while (true) {
+ if (CheckServiceHealth()) {
+ return;
+ }
+
+ Wait();
+ }
+ }
+
+ void EnsureServiceUnavailability() {
+ // check the response for 3 seconds
+ for (size_t i = 0; i < 30; ++i) {
+ UNIT_ASSERT(!CheckServiceHealth());
+ Wait();
+ }
+
+ TFancyGRpcWrapper(GrpcPort_).EnsureServiceUnavailability();
+ }
+
+private:
+ TPortManager PortManager_;
+ ui16 BusPort_;
+ ui16 GrpcPort_;
+ TServerSettings Settings_;
+
NMonitoring::TMetricRegistry Sensors_;
- THolder<TServer> Server_;
- THolder<NPersQueueTests::TFlatMsgBusPQClient> PQClient_;
-
+ THolder<TServer> Server_;
+ THolder<NPersQueueTests::TFlatMsgBusPQClient> PQClient_;
+
TActorId HttpProxyId_;
-
- THolder<TTempFileHandle> NetDataFile;
-};
-
-Y_UNIT_TEST_SUITE(TPQCDTest) {
- Y_UNIT_TEST(TestRelatedServicesAreRunning) {
- TPQCDServer server(false);
-
- const TString datacenterLabel = "fancy_datacenter";
-
- server.SetNetDataViaFile("2a0d:d6c0::/29\t" + datacenterLabel);
-
- server.Run();
-
- server.PQClient().InitRoot();
- server.PQClient().InitDCs();
-
- auto& actorSystem = server.ActorSystem();
- // check that NetClassifier is up and running
- {
+
+ THolder<TTempFileHandle> NetDataFile;
+};
+
+Y_UNIT_TEST_SUITE(TPQCDTest) {
+ Y_UNIT_TEST(TestRelatedServicesAreRunning) {
+ TPQCDServer server(false);
+
+ const TString datacenterLabel = "fancy_datacenter";
+
+ server.SetNetDataViaFile("2a0d:d6c0::/29\t" + datacenterLabel);
+
+ server.Run();
+
+ server.PQClient().InitRoot();
+ server.PQClient().InitDCs();
+
+ auto& actorSystem = server.ActorSystem();
+ // check that NetClassifier is up and running
+ {
const TActorId sender = actorSystem.AllocateEdgeActor();
-
- actorSystem.Send(
- new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender,
- new NNetClassifier::TEvNetClassifier::TEvSubscribe()
- ));
-
- TAutoPtr<IEventHandle> handle;
- const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(handle);
-
- UNIT_ASSERT(event->NetDataUpdateTimestamp);
- UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero());
-
- UNIT_ASSERT(event->Classifier);
- {
- const auto classificationResult = event->Classifier->ClassifyAddress("2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb");
-
- UNIT_ASSERT(classificationResult);
- UNIT_ASSERT_STRINGS_EQUAL(*classificationResult, datacenterLabel);
- }
-
- {
- const auto classificationResult = event->Classifier->ClassifyAddress("2b0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb");
- UNIT_ASSERT(!classificationResult);
- }
- }
-
- // check that ClusterTracker is up and running
- {
+
+ actorSystem.Send(
+ new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender,
+ new NNetClassifier::TEvNetClassifier::TEvSubscribe()
+ ));
+
+ TAutoPtr<IEventHandle> handle;
+ const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(handle);
+
+ UNIT_ASSERT(event->NetDataUpdateTimestamp);
+ UNIT_ASSERT(*event->NetDataUpdateTimestamp > TInstant::Zero());
+
+ UNIT_ASSERT(event->Classifier);
+ {
+ const auto classificationResult = event->Classifier->ClassifyAddress("2a0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb");
+
+ UNIT_ASSERT(classificationResult);
+ UNIT_ASSERT_STRINGS_EQUAL(*classificationResult, datacenterLabel);
+ }
+
+ {
+ const auto classificationResult = event->Classifier->ClassifyAddress("2b0d:d6c0:bbbb:bbbb:bbbb:bbbb:bbbb:bbbb");
+ UNIT_ASSERT(!classificationResult);
+ }
+ }
+
+ // check that ClusterTracker is up and running
+ {
const TActorId sender = actorSystem.AllocateEdgeActor();
-
- actorSystem.Send(
- new IEventHandle(NPQ::NClusterTracker::MakeClusterTrackerID(), sender,
- new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe()
- ));
-
- TAutoPtr<IEventHandle> handle;
- const auto event = actorSystem.GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>(handle);
-
- UNIT_ASSERT(event->ClustersList);
- UNIT_ASSERT(event->ClustersList->Clusters.size());
-
- UNIT_ASSERT(event->ClustersListUpdateTimestamp);
- UNIT_ASSERT(*event->ClustersListUpdateTimestamp > TInstant::Zero());
-
- UNIT_ASSERT_STRINGS_EQUAL(event->ClustersList->Clusters.front().Name, "dc1");
- }
- }
-
- Y_UNIT_TEST(TestDiscoverClusters) {
- TPQCDServer server;
-
- const TString datacenterLabel = "fancy_datacenter";
-
- server.SetNetDataViaFile("::1/128\t" + datacenterLabel);
-
- server.Run();
-
- server.PQClient().InitRoot();
- server.PQClient().InitDCs();
-
- server.WaitUntilHealthy();
-
- TFancyGRpcWrapper wrapper(server.GrpcPort());
-
- constexpr i64 dcTestIterations = 4; // it's signed to avoid warnings
- for (size_t i = 1; i <= dcTestIterations; ++i) {
- wrapper.WaitForExactClustersDataVersion(i);
-
- ClusterInfo c1Info;
- c1Info.set_name("dc1");
+
+ actorSystem.Send(
+ new IEventHandle(NPQ::NClusterTracker::MakeClusterTrackerID(), sender,
+ new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe()
+ ));
+
+ TAutoPtr<IEventHandle> handle;
+ const auto event = actorSystem.GrabEdgeEvent<NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate>(handle);
+
+ UNIT_ASSERT(event->ClustersList);
+ UNIT_ASSERT(event->ClustersList->Clusters.size());
+
+ UNIT_ASSERT(event->ClustersListUpdateTimestamp);
+ UNIT_ASSERT(*event->ClustersListUpdateTimestamp > TInstant::Zero());
+
+ UNIT_ASSERT_STRINGS_EQUAL(event->ClustersList->Clusters.front().Name, "dc1");
+ }
+ }
+
+ Y_UNIT_TEST(TestDiscoverClusters) {
+ TPQCDServer server;
+
+ const TString datacenterLabel = "fancy_datacenter";
+
+ server.SetNetDataViaFile("::1/128\t" + datacenterLabel);
+
+ server.Run();
+
+ server.PQClient().InitRoot();
+ server.PQClient().InitDCs();
+
+ server.WaitUntilHealthy();
+
+ TFancyGRpcWrapper wrapper(server.GrpcPort());
+
+ constexpr i64 dcTestIterations = 4; // it's signed to avoid warnings
+ for (size_t i = 1; i <= dcTestIterations; ++i) {
+ wrapper.WaitForExactClustersDataVersion(i);
+
+ ClusterInfo c1Info;
+ c1Info.set_name("dc1");
c1Info.set_endpoint("localhost");
- c1Info.set_available(true);
-
- ClusterInfo c2Info;
- c2Info.set_name("dc2");
- c2Info.set_endpoint("dc2.logbroker.yandex.net");
- c2Info.set_available(true);
-
- // atm it's impossible to disable reading
- CheckReadDiscoveryHandlers(wrapper, c1Info, c2Info, i);
-
- c1Info.set_available(i % 2);
-
- // only write discovery is affected
- CheckWriteDiscoveryHandlers(wrapper, c1Info, c2Info, i);
-
- server.PQClient().UpdateDC("dc1", true, !(i % 2));
- }
- {
- // test handle viewer
- auto responsePtr = server.HttpRequest("/actors/pqcd");
- const auto& response = *responsePtr->Get();
-
- UNIT_ASSERT(!response.Error);
- UNIT_ASSERT_STRINGS_EQUAL(response.Response->Status, "200");
- UNIT_ASSERT(response.Response->Body.Contains("dc1"));
- UNIT_ASSERT(response.Response->Body.Contains("dc2"));
- UNIT_ASSERT(response.Response->Body.Contains("NetClassifier"));
- UNIT_ASSERT(response.Response->Body.Contains("Clusters list"));
- UNIT_ASSERT(response.Response->Body.Contains("GOOD"));
- }
-
- UNIT_ASSERT_VALUES_EQUAL(server.ActorSystem().GetNodeCount(), 1);
-
- // counters values might be somehow higher since WaitForExactClustersDataVersion calls number is unpredictable
- // read check performs 4 calls, write check performs 2 calls. Also for each iteration WaitForExactClustersDataVersion is called at least once
- // therefore the final minimal expected calls number is (4 + 2 + 1) * dcTestIterations
- NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr);
- UNIT_ASSERT_GE(counters.TotalRequestsCount->GetAtomic(), (4 + 2 + 1) * dcTestIterations);
- UNIT_ASSERT_GE(counters.SuccessfulRequestsCount->GetAtomic(), (3 + 1 + 1) * dcTestIterations);
-
- UNIT_ASSERT_VALUES_EQUAL(counters.DroppedRequestsCount->GetAtomic(), 0);
- UNIT_ASSERT_VALUES_EQUAL(counters.HealthProbe->GetAtomic(), 1);
- UNIT_ASSERT_VALUES_EQUAL(counters.FailedRequestsCount->GetAtomic(), 0);
- UNIT_ASSERT_VALUES_EQUAL(counters.BadRequestsCount->GetAtomic(), 2 * dcTestIterations);
-
- UNIT_ASSERT_VALUES_EQUAL(counters.WriteDiscoveriesCount->GetAtomic(), 4 * dcTestIterations);
- UNIT_ASSERT_VALUES_EQUAL(counters.ReadDiscoveriesCount->GetAtomic(), 3 * dcTestIterations);
-
- auto getTimesPrioritized = [&server](const TString& clusterName) {
- return server.ServiceCounters()->GetSubgroup("prioritized_cluster", clusterName)->GetCounter("TimesPrioritizedForWrite", true)->GetAtomic();
- };
-
- UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc1"), 1 * dcTestIterations);
- UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc2"), 2 * dcTestIterations);
-
- // all requests are expected to come from a certain dc since the locality of the test
- UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier(datacenterLabel), counters.TotalRequestsCount->GetAtomic());
-
- auto snapshot = counters.DiscoveryWorkingDurationMs->Snapshot();
-
- size_t total = 0;
- for (size_t i = 0; i < snapshot->Count(); ++i) {
- total += snapshot->Value(i);
- }
- UNIT_ASSERT_VALUES_EQUAL(total, counters.TotalRequestsCount->GetAtomic() - counters.DroppedRequestsCount->GetAtomic());
-
- UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 0);
-
- UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_PREFERENCE]->GetAtomic(), 1 * dcTestIterations);
- UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0);
- UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 2 * dcTestIterations);
- }
-
- Y_UNIT_TEST(TestPrioritizeLocalDatacenter) {
- TPQCDServer server;
-
- const TString datacenterLabel = "dc2";
-
- server.SetNetDataViaFile("::1/128\t" + datacenterLabel);
-
- server.Run();
-
- server.PQClient().InitRoot();
- server.PQClient().InitDCs();
-
- server.WaitUntilHealthy();
-
- TFancyGRpcWrapper wrapper(server.GrpcPort());
-
- DiscoverClustersRequest request;
- {
- auto* writeSessionParams = request.add_write_sessions();
- writeSessionParams->set_topic("topic");
- writeSessionParams->set_source_id("topic1.log");
- writeSessionParams->set_partition_group(42);
- }
-
- DiscoverClustersResult result;
-
- CallPQCDAndAssert(wrapper, request, result, 1, 1, 0);
-
- const auto& writeSessionClusters = result.write_sessions_clusters(0);
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
-
- UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), datacenterLabel);
-
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_LOCATION);
-
- NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr);
-
- UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 1);
- UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 0);
- }
-
- Y_UNIT_TEST(TestUnavailableWithoutNetClassifier) {
- TPQCDServer server;
-
- server.Run();
-
- server.PQClient().InitRoot();
- server.PQClient().InitDCs();
-
- auto& actorSystem = server.ActorSystem();
- // check that NetClassifier instance is initialized with null
- {
+ c1Info.set_available(true);
+
+ ClusterInfo c2Info;
+ c2Info.set_name("dc2");
+ c2Info.set_endpoint("dc2.logbroker.yandex.net");
+ c2Info.set_available(true);
+
+ // atm it's impossible to disable reading
+ CheckReadDiscoveryHandlers(wrapper, c1Info, c2Info, i);
+
+ c1Info.set_available(i % 2);
+
+ // only write discovery is affected
+ CheckWriteDiscoveryHandlers(wrapper, c1Info, c2Info, i);
+
+ server.PQClient().UpdateDC("dc1", true, !(i % 2));
+ }
+ {
+ // test handle viewer
+ auto responsePtr = server.HttpRequest("/actors/pqcd");
+ const auto& response = *responsePtr->Get();
+
+ UNIT_ASSERT(!response.Error);
+ UNIT_ASSERT_STRINGS_EQUAL(response.Response->Status, "200");
+ UNIT_ASSERT(response.Response->Body.Contains("dc1"));
+ UNIT_ASSERT(response.Response->Body.Contains("dc2"));
+ UNIT_ASSERT(response.Response->Body.Contains("NetClassifier"));
+ UNIT_ASSERT(response.Response->Body.Contains("Clusters list"));
+ UNIT_ASSERT(response.Response->Body.Contains("GOOD"));
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(server.ActorSystem().GetNodeCount(), 1);
+
+ // counters values might be somehow higher since WaitForExactClustersDataVersion calls number is unpredictable
+ // read check performs 4 calls, write check performs 2 calls. Also for each iteration WaitForExactClustersDataVersion is called at least once
+ // therefore the final minimal expected calls number is (4 + 2 + 1) * dcTestIterations
+ NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr);
+ UNIT_ASSERT_GE(counters.TotalRequestsCount->GetAtomic(), (4 + 2 + 1) * dcTestIterations);
+ UNIT_ASSERT_GE(counters.SuccessfulRequestsCount->GetAtomic(), (3 + 1 + 1) * dcTestIterations);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.DroppedRequestsCount->GetAtomic(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(counters.HealthProbe->GetAtomic(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(counters.FailedRequestsCount->GetAtomic(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(counters.BadRequestsCount->GetAtomic(), 2 * dcTestIterations);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.WriteDiscoveriesCount->GetAtomic(), 4 * dcTestIterations);
+ UNIT_ASSERT_VALUES_EQUAL(counters.ReadDiscoveriesCount->GetAtomic(), 3 * dcTestIterations);
+
+ auto getTimesPrioritized = [&server](const TString& clusterName) {
+ return server.ServiceCounters()->GetSubgroup("prioritized_cluster", clusterName)->GetCounter("TimesPrioritizedForWrite", true)->GetAtomic();
+ };
+
+ UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc1"), 1 * dcTestIterations);
+ UNIT_ASSERT_VALUES_EQUAL(getTimesPrioritized("dc2"), 2 * dcTestIterations);
+
+ // all requests are expected to come from a certain dc since the locality of the test
+ UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier(datacenterLabel), counters.TotalRequestsCount->GetAtomic());
+
+ auto snapshot = counters.DiscoveryWorkingDurationMs->Snapshot();
+
+ size_t total = 0;
+ for (size_t i = 0; i < snapshot->Count(); ++i) {
+ total += snapshot->Value(i);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(total, counters.TotalRequestsCount->GetAtomic() - counters.DroppedRequestsCount->GetAtomic());
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_PREFERENCE]->GetAtomic(), 1 * dcTestIterations);
+ UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 2 * dcTestIterations);
+ }
+
+ Y_UNIT_TEST(TestPrioritizeLocalDatacenter) {
+ TPQCDServer server;
+
+ const TString datacenterLabel = "dc2";
+
+ server.SetNetDataViaFile("::1/128\t" + datacenterLabel);
+
+ server.Run();
+
+ server.PQClient().InitRoot();
+ server.PQClient().InitDCs();
+
+ server.WaitUntilHealthy();
+
+ TFancyGRpcWrapper wrapper(server.GrpcPort());
+
+ DiscoverClustersRequest request;
+ {
+ auto* writeSessionParams = request.add_write_sessions();
+ writeSessionParams->set_topic("topic");
+ writeSessionParams->set_source_id("topic1.log");
+ writeSessionParams->set_partition_group(42);
+ }
+
+ DiscoverClustersResult result;
+
+ CallPQCDAndAssert(wrapper, request, result, 1, 1, 0);
+
+ const auto& writeSessionClusters = result.write_sessions_clusters(0);
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
+
+ UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), datacenterLabel);
+
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CLIENT_LOCATION);
+
+ NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 0);
+ }
+
+ Y_UNIT_TEST(TestUnavailableWithoutNetClassifier) {
+ TPQCDServer server;
+
+ server.Run();
+
+ server.PQClient().InitRoot();
+ server.PQClient().InitDCs();
+
+ auto& actorSystem = server.ActorSystem();
+ // check that NetClassifier instance is initialized with null
+ {
const TActorId sender = actorSystem.AllocateEdgeActor();
-
- actorSystem.Send(
- new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender,
- new NNetClassifier::TEvNetClassifier::TEvSubscribe()
- ));
-
- TAutoPtr<IEventHandle> handle;
- const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(sender);
- UNIT_ASSERT(!event->Get()->Classifier);
- }
-
- server.EnsureServiceUnavailability();
- }
-
- Y_UNIT_TEST(TestUnavailableWithoutClustersList) {
- TPQCDServer server;
-
- server.SetNetDataViaFile("::1/128\tdc1");
-
- server.Run();
-
- server.EnsureServiceUnavailability();
- }
-
- Y_UNIT_TEST(TestUnavailableWithoutBoth) {
- TPQCDServer server;
-
- server.Run();
-
- server.EnsureServiceUnavailability();
- }
-
- Y_UNIT_TEST(TestCloudClientsAreConsistentlyDistributed) {
- TPQCDServer server;
-
- auto& cloudNets = *server.MutableSettings().PQClusterDiscoveryConfig.MutableCloudNetData();
- auto& subnet = *cloudNets.AddSubnets();
- subnet.SetMask("::1/128");
- subnet.SetLabel("cloudnets");
-
- server.SetNetDataViaFile("::1/128\tdc2");
-
- server.Run();
-
- server.PQClient().InitRoot();
- server.PQClient().InitDCs();
-
- server.WaitUntilHealthy();
-
- TFancyGRpcWrapper wrapper(server.GrpcPort());
-
- DiscoverClustersRequest request;
- {
- auto* writeSessionParams = request.add_write_sessions();
- writeSessionParams->set_topic("topic12");
- writeSessionParams->set_source_id("topic1.log");
- }
-
- DiscoverClustersResult result;
-
- CallPQCDAndAssert(wrapper, request, result, 1, 1, 0);
-
- const auto& writeSessionClusters = result.write_sessions_clusters(0);
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
-
- UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), "dc1");
-
- UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION);
-
- NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr);
-
- UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 1);
- UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier("dc2"), 1);
-
- UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0);
- UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 1);
- }
-}
-
-} // namespace NKikimr::NPQCDTests
+
+ actorSystem.Send(
+ new IEventHandle(NNetClassifier::MakeNetClassifierID(), sender,
+ new NNetClassifier::TEvNetClassifier::TEvSubscribe()
+ ));
+
+ TAutoPtr<IEventHandle> handle;
+ const auto event = actorSystem.GrabEdgeEvent<NNetClassifier::TEvNetClassifier::TEvClassifierUpdate>(sender);
+ UNIT_ASSERT(!event->Get()->Classifier);
+ }
+
+ server.EnsureServiceUnavailability();
+ }
+
+ Y_UNIT_TEST(TestUnavailableWithoutClustersList) {
+ TPQCDServer server;
+
+ server.SetNetDataViaFile("::1/128\tdc1");
+
+ server.Run();
+
+ server.EnsureServiceUnavailability();
+ }
+
+ Y_UNIT_TEST(TestUnavailableWithoutBoth) {
+ TPQCDServer server;
+
+ server.Run();
+
+ server.EnsureServiceUnavailability();
+ }
+
+ Y_UNIT_TEST(TestCloudClientsAreConsistentlyDistributed) {
+ TPQCDServer server;
+
+ auto& cloudNets = *server.MutableSettings().PQClusterDiscoveryConfig.MutableCloudNetData();
+ auto& subnet = *cloudNets.AddSubnets();
+ subnet.SetMask("::1/128");
+ subnet.SetLabel("cloudnets");
+
+ server.SetNetDataViaFile("::1/128\tdc2");
+
+ server.Run();
+
+ server.PQClient().InitRoot();
+ server.PQClient().InitDCs();
+
+ server.WaitUntilHealthy();
+
+ TFancyGRpcWrapper wrapper(server.GrpcPort());
+
+ DiscoverClustersRequest request;
+ {
+ auto* writeSessionParams = request.add_write_sessions();
+ writeSessionParams->set_topic("topic12");
+ writeSessionParams->set_source_id("topic1.log");
+ }
+
+ DiscoverClustersResult result;
+
+ CallPQCDAndAssert(wrapper, request, result, 1, 1, 0);
+
+ const auto& writeSessionClusters = result.write_sessions_clusters(0);
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.clusters_size(), 2);
+
+ UNIT_ASSERT_STRINGS_EQUAL(writeSessionClusters.clusters(0).name(), "dc1");
+
+ UNIT_ASSERT_VALUES_EQUAL(writeSessionClusters.primary_cluster_selection_reason(), WriteSessionClusters::CONSISTENT_DISTRIBUTION);
+
+ NPQ::NClusterDiscovery::NCounters::TClusterDiscoveryCounters counters(server.ServiceCounters(), nullptr, nullptr);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.InfracloudRequestsCount->GetAtomic(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(server.GetTimesResolvedByClassifier("dc2"), 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CLIENT_LOCATION]->GetAtomic(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(counters.PrimaryClusterSelectionReasons[WriteSessionClusters::CONSISTENT_DISTRIBUTION]->GetAtomic(), 1);
+ }
+}
+
+} // namespace NKikimr::NPQCDTests
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp
index 5882832912..0c061b9bea 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp
+++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.cpp
@@ -1,288 +1,288 @@
-#include "cluster_discovery_worker.h"
-
+#include "cluster_discovery_worker.h"
+
#include <ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <library/cpp/actors/core/hfunc.h>
-
-#include <util/digest/numeric.h>
-#include <util/generic/hash.h>
-#include <util/string/ascii.h>
-
-#include <algorithm>
-
-
-namespace NKikimr::NPQ::NClusterDiscovery::NWorker {
-
-using namespace Ydb::PersQueue::ClusterDiscovery;
-using NKikimr::NPQ::NClusterDiscovery::NClusterOrdering::OrderByHashAndWeight;
-
-inline auto& Ctx() {
- return TActivationContext::AsActorContext();
-}
-
-class TClusterDiscoveryWorker : public TActorBootstrapped<TClusterDiscoveryWorker> {
-public:
- static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
- return NKikimrServices::TActivity::GRPC_REQ;
- }
-
- TClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev,
- TLabeledAddressClassifier::TConstPtr datacenterClassifier,
- TLabeledAddressClassifier::TConstPtr cloudNetsClassifier,
- TClustersList::TConstPtr clustersList,
- TClusterDiscoveryCounters::TPtr counters)
- : Request(ev->Release().Release())
- , DatacenterClassifier(std::move(datacenterClassifier))
- , CloudNetsClassifier(std::move(cloudNetsClassifier))
- , ClustersList(std::move(clustersList))
- , Counters(counters)
- {}
-
- void Bootstrap() {
- RequestStartTs = Ctx().Now();
- Process();
- }
-
- template<typename TSelectClusterFunc>
- bool MoveTheBestClusterToFront(std::vector<TClustersList::TCluster>& clusters, const TSelectClusterFunc selectClusterFunc) const {
- auto it = std::find_if(begin(clusters), end(clusters), selectClusterFunc);
-
- if (it == clusters.end()) {
- return false;
- } else if (it != begin(clusters)) {
- std::swap(*begin(clusters), *it);
- }
-
- return true;
- }
-
- bool FillWriteSessionClusters(const TClustersList& clustersList,
- const WriteSessionParams& sessionParams,
- WriteSessionClusters& sessionClusters) const
- {
- auto clusters = clustersList.Clusters; // make a copy for reordering
-
- bool movedFirstPriorityCluster = false;
- WriteSessionClusters::SelectionReason primaryClusterSelectionReason = WriteSessionClusters::CONSISTENT_DISTRIBUTION;
-
- if (sessionParams.preferred_cluster_name()) {
- movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters,
- [preferred_cluster_name = sessionParams.preferred_cluster_name()](const auto& cluster) {
- return AsciiEqualsIgnoreCase(cluster.Name, preferred_cluster_name);
- }
- );
-
- if (!movedFirstPriorityCluster) {
- return false; // failed to prioritize the specified cluster
- }
-
- primaryClusterSelectionReason = WriteSessionClusters::CLIENT_PREFERENCE;
- }
-
- if (!movedFirstPriorityCluster && ClientDatacenterName && !IsInfracloudClient) {
- movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters,
- [datacenter = *ClientDatacenterName](const auto& cluster) {
- return AsciiEqualsIgnoreCase(cluster.Datacenter, datacenter);
- }
- );
-
- if (movedFirstPriorityCluster) {
- primaryClusterSelectionReason = WriteSessionClusters::CLIENT_LOCATION;
- }
- }
-
- ui64 hashValue = THash<TString>()(sessionParams.topic());
- hashValue = CombineHashes(hashValue, THash<TString>()(sessionParams.source_id()));
- hashValue = CombineHashes(hashValue, static_cast<ui64>(sessionParams.partition_group()));
-
- auto reorderingBeginIt = begin(clusters);
- if (!clusters.empty() && movedFirstPriorityCluster) {
- ++reorderingBeginIt;
- }
-
- OrderByHashAndWeight(reorderingBeginIt, end(clusters), hashValue, [](const auto& cluster){ return cluster.Weight; });
-
- sessionClusters.set_primary_cluster_selection_reason(primaryClusterSelectionReason);
- ReportPrimaryClusterSelectionReason(primaryClusterSelectionReason);
-
- if (!clusters.empty()) {
- ReportPrioritizedCluster(clusters.front());
- }
-
- for (const auto& cluster : clusters) {
- auto& clusterInfo = *sessionClusters.add_clusters();
- clusterInfo.set_endpoint(cluster.Balancer);
- clusterInfo.set_name(cluster.Name);
- clusterInfo.set_available(cluster.IsEnabled);
- }
-
- return true;
- }
-
- bool FillReadSessionClusters(const TClustersList& clustersList,
- const ReadSessionParams& sessionParams,
- ReadSessionClusters& sessionClusters) const
- {
- if (sessionParams.has_all_original()) {
- for (const auto& cluster : clustersList.Clusters) {
- auto& clusterInfo = *sessionClusters.add_clusters();
- clusterInfo.set_endpoint(cluster.Balancer);
- clusterInfo.set_name(cluster.Name);
- clusterInfo.set_available(true); // at the moment we can't logically disable reading
- }
- } else {
- auto it = std::find_if(begin(clustersList.Clusters), end(clustersList.Clusters),
- [mirror_to_cluster = sessionParams.mirror_to_cluster()](const auto& cluster) {
- return AsciiEqualsIgnoreCase(cluster.Name, mirror_to_cluster);
- }
- );
-
- if (it != end(clustersList.Clusters)) {
- auto& clusterInfo = *sessionClusters.add_clusters();
- clusterInfo.set_endpoint(it->Balancer);
- clusterInfo.set_name(it->Name);
- clusterInfo.set_available(true);
- } else {
- return false;
- }
- }
-
- return true;
- }
-
- TMaybe<TString> TryDetectDatacenter(const TString& address) const {
- if (!DatacenterClassifier) {
- return Nothing();
- }
-
- return DatacenterClassifier->ClassifyAddress(address);
- }
-
- void Process() {
+
+#include <util/digest/numeric.h>
+#include <util/generic/hash.h>
+#include <util/string/ascii.h>
+
+#include <algorithm>
+
+
+namespace NKikimr::NPQ::NClusterDiscovery::NWorker {
+
+using namespace Ydb::PersQueue::ClusterDiscovery;
+using NKikimr::NPQ::NClusterDiscovery::NClusterOrdering::OrderByHashAndWeight;
+
+inline auto& Ctx() {
+ return TActivationContext::AsActorContext();
+}
+
+class TClusterDiscoveryWorker : public TActorBootstrapped<TClusterDiscoveryWorker> {
+public:
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::GRPC_REQ;
+ }
+
+ TClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev,
+ TLabeledAddressClassifier::TConstPtr datacenterClassifier,
+ TLabeledAddressClassifier::TConstPtr cloudNetsClassifier,
+ TClustersList::TConstPtr clustersList,
+ TClusterDiscoveryCounters::TPtr counters)
+ : Request(ev->Release().Release())
+ , DatacenterClassifier(std::move(datacenterClassifier))
+ , CloudNetsClassifier(std::move(cloudNetsClassifier))
+ , ClustersList(std::move(clustersList))
+ , Counters(counters)
+ {}
+
+ void Bootstrap() {
+ RequestStartTs = Ctx().Now();
+ Process();
+ }
+
+ template<typename TSelectClusterFunc>
+ bool MoveTheBestClusterToFront(std::vector<TClustersList::TCluster>& clusters, const TSelectClusterFunc selectClusterFunc) const {
+ auto it = std::find_if(begin(clusters), end(clusters), selectClusterFunc);
+
+ if (it == clusters.end()) {
+ return false;
+ } else if (it != begin(clusters)) {
+ std::swap(*begin(clusters), *it);
+ }
+
+ return true;
+ }
+
+ bool FillWriteSessionClusters(const TClustersList& clustersList,
+ const WriteSessionParams& sessionParams,
+ WriteSessionClusters& sessionClusters) const
+ {
+ auto clusters = clustersList.Clusters; // make a copy for reordering
+
+ bool movedFirstPriorityCluster = false;
+ WriteSessionClusters::SelectionReason primaryClusterSelectionReason = WriteSessionClusters::CONSISTENT_DISTRIBUTION;
+
+ if (sessionParams.preferred_cluster_name()) {
+ movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters,
+ [preferred_cluster_name = sessionParams.preferred_cluster_name()](const auto& cluster) {
+ return AsciiEqualsIgnoreCase(cluster.Name, preferred_cluster_name);
+ }
+ );
+
+ if (!movedFirstPriorityCluster) {
+ return false; // failed to prioritize the specified cluster
+ }
+
+ primaryClusterSelectionReason = WriteSessionClusters::CLIENT_PREFERENCE;
+ }
+
+ if (!movedFirstPriorityCluster && ClientDatacenterName && !IsInfracloudClient) {
+ movedFirstPriorityCluster = MoveTheBestClusterToFront(clusters,
+ [datacenter = *ClientDatacenterName](const auto& cluster) {
+ return AsciiEqualsIgnoreCase(cluster.Datacenter, datacenter);
+ }
+ );
+
+ if (movedFirstPriorityCluster) {
+ primaryClusterSelectionReason = WriteSessionClusters::CLIENT_LOCATION;
+ }
+ }
+
+ ui64 hashValue = THash<TString>()(sessionParams.topic());
+ hashValue = CombineHashes(hashValue, THash<TString>()(sessionParams.source_id()));
+ hashValue = CombineHashes(hashValue, static_cast<ui64>(sessionParams.partition_group()));
+
+ auto reorderingBeginIt = begin(clusters);
+ if (!clusters.empty() && movedFirstPriorityCluster) {
+ ++reorderingBeginIt;
+ }
+
+ OrderByHashAndWeight(reorderingBeginIt, end(clusters), hashValue, [](const auto& cluster){ return cluster.Weight; });
+
+ sessionClusters.set_primary_cluster_selection_reason(primaryClusterSelectionReason);
+ ReportPrimaryClusterSelectionReason(primaryClusterSelectionReason);
+
+ if (!clusters.empty()) {
+ ReportPrioritizedCluster(clusters.front());
+ }
+
+ for (const auto& cluster : clusters) {
+ auto& clusterInfo = *sessionClusters.add_clusters();
+ clusterInfo.set_endpoint(cluster.Balancer);
+ clusterInfo.set_name(cluster.Name);
+ clusterInfo.set_available(cluster.IsEnabled);
+ }
+
+ return true;
+ }
+
+ bool FillReadSessionClusters(const TClustersList& clustersList,
+ const ReadSessionParams& sessionParams,
+ ReadSessionClusters& sessionClusters) const
+ {
+ if (sessionParams.has_all_original()) {
+ for (const auto& cluster : clustersList.Clusters) {
+ auto& clusterInfo = *sessionClusters.add_clusters();
+ clusterInfo.set_endpoint(cluster.Balancer);
+ clusterInfo.set_name(cluster.Name);
+ clusterInfo.set_available(true); // at the moment we can't logically disable reading
+ }
+ } else {
+ auto it = std::find_if(begin(clustersList.Clusters), end(clustersList.Clusters),
+ [mirror_to_cluster = sessionParams.mirror_to_cluster()](const auto& cluster) {
+ return AsciiEqualsIgnoreCase(cluster.Name, mirror_to_cluster);
+ }
+ );
+
+ if (it != end(clustersList.Clusters)) {
+ auto& clusterInfo = *sessionClusters.add_clusters();
+ clusterInfo.set_endpoint(it->Balancer);
+ clusterInfo.set_name(it->Name);
+ clusterInfo.set_available(true);
+ } else {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ TMaybe<TString> TryDetectDatacenter(const TString& address) const {
+ if (!DatacenterClassifier) {
+ return Nothing();
+ }
+
+ return DatacenterClassifier->ClassifyAddress(address);
+ }
+
+ void Process() {
auto* result = TEvDiscoverPQClustersRequest::AllocateResult<DiscoverClustersResult>(Request);
-
- auto statusCode = Ydb::StatusIds::INTERNAL_ERROR;
-
- if (ClustersList) {
- const TString address = NAddressClassifier::ExtractAddress(Request->GetPeerName());
-
- ClientDatacenterName = TryDetectDatacenter(address);
-
- if (CloudNetsClassifier && CloudNetsClassifier->ClassifyAddress(address)) {
- IsInfracloudClient = true;
- }
-
- statusCode = ProcessWriteSessions(*result);
- if (statusCode == Ydb::StatusIds::SUCCESS) {
- statusCode = ProcessReadSessions(*result);
- }
-
- result->set_version(ClustersList->Version);
- }
-
- ReportMetrics(statusCode);
-
- Request->SendResult(*result, statusCode);
-
- return PassAway();
- }
-
- Ydb::StatusIds::StatusCode ProcessWriteSessions(DiscoverClustersResult& result) const {
- const size_t sessionsCount = Request->GetProtoRequest()->write_sessions_size();
- for (size_t i = 0; i < sessionsCount; ++i) {
- if (!FillWriteSessionClusters(*ClustersList,
- Request->GetProtoRequest()->write_sessions(i),
- *result.add_write_sessions_clusters()))
- {
- return Ydb::StatusIds::BAD_REQUEST;
- }
- }
-
- return Ydb::StatusIds::SUCCESS;
- }
-
- Ydb::StatusIds::StatusCode ProcessReadSessions(DiscoverClustersResult& result) const {
- const size_t sessionsCount = Request->GetProtoRequest()->read_sessions_size();
- for (size_t i = 0; i < sessionsCount; ++i) {
- if (!FillReadSessionClusters(*ClustersList,
- Request->GetProtoRequest()->read_sessions(i),
- *result.add_read_sessions_clusters()))
- {
- return Ydb::StatusIds::BAD_REQUEST;
- }
- }
-
- return Ydb::StatusIds::SUCCESS;
- }
-
- // Monitoring
-
- template<typename TMappedCounters, typename TLabel>
- void IncrementLabeledCounter(TMappedCounters& counters, const TLabel& label) const {
- auto it = counters.find(label);
-
- Y_VERIFY(it != counters.end());
-
- it->second->Inc();
- }
-
- void ReportPrioritizedCluster(const TClustersList::TCluster& cluster) const {
- IncrementLabeledCounter(Counters->TimesPrioritizedForWrite, cluster.Name);
- }
-
- void ReportPrimaryClusterSelectionReason(const WriteSessionClusters::SelectionReason reason) const {
- IncrementLabeledCounter(Counters->PrimaryClusterSelectionReasons, reason);
- }
-
- void ReportMetrics(const Ydb::StatusIds::StatusCode statusCode) const {
- if (statusCode == Ydb::StatusIds::SUCCESS) {
- Counters->SuccessfulRequestsCount->Inc();
- } else if (statusCode == Ydb::StatusIds::BAD_REQUEST) {
- Counters->BadRequestsCount->Inc();
- } else {
- Y_VERIFY(statusCode == Ydb::StatusIds::INTERNAL_ERROR);
- Counters->FailedRequestsCount->Inc();
- }
-
- Counters->WriteDiscoveriesCount->Add(Request->GetProtoRequest()->write_sessions_size());
- Counters->ReadDiscoveriesCount->Add(Request->GetProtoRequest()->read_sessions_size());
-
- if (ClientDatacenterName) {
- IncrementLabeledCounter(Counters->TimesResolvedByClassifier, *ClientDatacenterName);
- }
-
- if (IsInfracloudClient) {
- Counters->InfracloudRequestsCount->Inc();
- }
-
- Counters->DiscoveryWorkingDurationMs->Collect(Ctx().Now().MilliSeconds() - RequestStartTs.MilliSeconds());
- }
-
-private:
- THolder<TEvDiscoverPQClustersRequest> Request;
- TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null
- TLabeledAddressClassifier::TConstPtr CloudNetsClassifier; // Special classifier instance for cloud networks detection. May be null
- TMaybe<TString> ClientDatacenterName; // Detected client datacenter
- bool IsInfracloudClient = false; // Client came from infracloud
- TClustersList::TConstPtr ClustersList; // Cached list of clusters
- TInstant RequestStartTs;
-
- TClusterDiscoveryCounters::TPtr Counters;
-};
-
-IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev,
- TLabeledAddressClassifier::TConstPtr datacenterClassifier,
- TLabeledAddressClassifier::TConstPtr cloudNetsClassifier,
- TClustersList::TConstPtr clustersList,
- TClusterDiscoveryCounters::TPtr counters)
-{
- return new TClusterDiscoveryWorker(ev,
- std::move(datacenterClassifier),
- std::move(cloudNetsClassifier),
- std::move(clustersList),
- std::move(counters)
- );
-}
-
-} // namespace NKikimr::NPQ::NClusterDiscovery::NWorker
+
+ auto statusCode = Ydb::StatusIds::INTERNAL_ERROR;
+
+ if (ClustersList) {
+ const TString address = NAddressClassifier::ExtractAddress(Request->GetPeerName());
+
+ ClientDatacenterName = TryDetectDatacenter(address);
+
+ if (CloudNetsClassifier && CloudNetsClassifier->ClassifyAddress(address)) {
+ IsInfracloudClient = true;
+ }
+
+ statusCode = ProcessWriteSessions(*result);
+ if (statusCode == Ydb::StatusIds::SUCCESS) {
+ statusCode = ProcessReadSessions(*result);
+ }
+
+ result->set_version(ClustersList->Version);
+ }
+
+ ReportMetrics(statusCode);
+
+ Request->SendResult(*result, statusCode);
+
+ return PassAway();
+ }
+
+ Ydb::StatusIds::StatusCode ProcessWriteSessions(DiscoverClustersResult& result) const {
+ const size_t sessionsCount = Request->GetProtoRequest()->write_sessions_size();
+ for (size_t i = 0; i < sessionsCount; ++i) {
+ if (!FillWriteSessionClusters(*ClustersList,
+ Request->GetProtoRequest()->write_sessions(i),
+ *result.add_write_sessions_clusters()))
+ {
+ return Ydb::StatusIds::BAD_REQUEST;
+ }
+ }
+
+ return Ydb::StatusIds::SUCCESS;
+ }
+
+ Ydb::StatusIds::StatusCode ProcessReadSessions(DiscoverClustersResult& result) const {
+ const size_t sessionsCount = Request->GetProtoRequest()->read_sessions_size();
+ for (size_t i = 0; i < sessionsCount; ++i) {
+ if (!FillReadSessionClusters(*ClustersList,
+ Request->GetProtoRequest()->read_sessions(i),
+ *result.add_read_sessions_clusters()))
+ {
+ return Ydb::StatusIds::BAD_REQUEST;
+ }
+ }
+
+ return Ydb::StatusIds::SUCCESS;
+ }
+
+ // Monitoring
+
+ template<typename TMappedCounters, typename TLabel>
+ void IncrementLabeledCounter(TMappedCounters& counters, const TLabel& label) const {
+ auto it = counters.find(label);
+
+ Y_VERIFY(it != counters.end());
+
+ it->second->Inc();
+ }
+
+ void ReportPrioritizedCluster(const TClustersList::TCluster& cluster) const {
+ IncrementLabeledCounter(Counters->TimesPrioritizedForWrite, cluster.Name);
+ }
+
+ void ReportPrimaryClusterSelectionReason(const WriteSessionClusters::SelectionReason reason) const {
+ IncrementLabeledCounter(Counters->PrimaryClusterSelectionReasons, reason);
+ }
+
+ void ReportMetrics(const Ydb::StatusIds::StatusCode statusCode) const {
+ if (statusCode == Ydb::StatusIds::SUCCESS) {
+ Counters->SuccessfulRequestsCount->Inc();
+ } else if (statusCode == Ydb::StatusIds::BAD_REQUEST) {
+ Counters->BadRequestsCount->Inc();
+ } else {
+ Y_VERIFY(statusCode == Ydb::StatusIds::INTERNAL_ERROR);
+ Counters->FailedRequestsCount->Inc();
+ }
+
+ Counters->WriteDiscoveriesCount->Add(Request->GetProtoRequest()->write_sessions_size());
+ Counters->ReadDiscoveriesCount->Add(Request->GetProtoRequest()->read_sessions_size());
+
+ if (ClientDatacenterName) {
+ IncrementLabeledCounter(Counters->TimesResolvedByClassifier, *ClientDatacenterName);
+ }
+
+ if (IsInfracloudClient) {
+ Counters->InfracloudRequestsCount->Inc();
+ }
+
+ Counters->DiscoveryWorkingDurationMs->Collect(Ctx().Now().MilliSeconds() - RequestStartTs.MilliSeconds());
+ }
+
+private:
+ THolder<TEvDiscoverPQClustersRequest> Request;
+ TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null
+ TLabeledAddressClassifier::TConstPtr CloudNetsClassifier; // Special classifier instance for cloud networks detection. May be null
+ TMaybe<TString> ClientDatacenterName; // Detected client datacenter
+ bool IsInfracloudClient = false; // Client came from infracloud
+ TClustersList::TConstPtr ClustersList; // Cached list of clusters
+ TInstant RequestStartTs;
+
+ TClusterDiscoveryCounters::TPtr Counters;
+};
+
+IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev,
+ TLabeledAddressClassifier::TConstPtr datacenterClassifier,
+ TLabeledAddressClassifier::TConstPtr cloudNetsClassifier,
+ TClustersList::TConstPtr clustersList,
+ TClusterDiscoveryCounters::TPtr counters)
+{
+ return new TClusterDiscoveryWorker(ev,
+ std::move(datacenterClassifier),
+ std::move(cloudNetsClassifier),
+ std::move(clustersList),
+ std::move(counters)
+ );
+}
+
+} // namespace NKikimr::NPQ::NClusterDiscovery::NWorker
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h
index 65cfe165f8..c92bb64940 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h
+++ b/ydb/services/persqueue_cluster_discovery/cluster_discovery_worker.h
@@ -1,25 +1,25 @@
-#pragma once
-
-#include "counters.h"
-
+#pragma once
+
+#include "counters.h"
+
#include <ydb/core/grpc_services/grpc_request_proxy.h>
#include <ydb/core/persqueue/cluster_tracker.h>
#include <ydb/core/util/address_classifier.h>
-
+
#include <library/cpp/actors/core/actor_bootstrapped.h>
-
-namespace NKikimr::NPQ::NClusterDiscovery::NWorker {
-
-using namespace NActors;
-using namespace NAddressClassifier;
-using namespace NGRpcService;
-using namespace NPQ::NClusterTracker;
-using namespace NCounters;
-
-IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev,
- TLabeledAddressClassifier::TConstPtr datacenterClassifier,
- TLabeledAddressClassifier::TConstPtr cloudNetsClassifier,
- TClustersList::TConstPtr clustersList,
- TClusterDiscoveryCounters::TPtr counters);
-
-} // namespace NKikimr::NClusterDiscovery::NWorker
+
+namespace NKikimr::NPQ::NClusterDiscovery::NWorker {
+
+using namespace NActors;
+using namespace NAddressClassifier;
+using namespace NGRpcService;
+using namespace NPQ::NClusterTracker;
+using namespace NCounters;
+
+IActor* CreateClusterDiscoveryWorker(TEvDiscoverPQClustersRequest::TPtr& ev,
+ TLabeledAddressClassifier::TConstPtr datacenterClassifier,
+ TLabeledAddressClassifier::TConstPtr cloudNetsClassifier,
+ TClustersList::TConstPtr clustersList,
+ TClusterDiscoveryCounters::TPtr counters);
+
+} // namespace NKikimr::NClusterDiscovery::NWorker
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make
index f7863157f3..c1e137b002 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ut/ya.make
@@ -1,17 +1,17 @@
UNITTEST_FOR(ydb/services/persqueue_cluster_discovery/cluster_ordering)
-
-OWNER(
- radix
+
+OWNER(
+ radix
g:kikimr
g:logbroker
-)
-
-SRCS(
- weighed_ordering_ut.cpp
-)
-
-PEERDIR(
+)
+
+SRCS(
+ weighed_ordering_ut.cpp
+)
+
+PEERDIR(
ydb/services/persqueue_cluster_discovery/cluster_ordering
-)
-
-END()
+)
+
+END()
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp
index 35c0234095..be1823bcff 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp
+++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.cpp
@@ -1 +1 @@
-#include "weighed_ordering.h"
+#include "weighed_ordering.h"
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h
index 13b045cf5e..a9695507ec 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h
+++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h
@@ -1,50 +1,50 @@
-#pragma once
-
-#include <algorithm>
-#include <type_traits>
-#include <vector>
-
-#include <util/system/types.h>
+#pragma once
+
+#include <algorithm>
+#include <type_traits>
+#include <vector>
+
+#include <util/system/types.h>
#include <util/system/yassert.h>
-
-namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering {
-
-template <typename Iterator, typename GetWeightFunc>
-Iterator SelectByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) {
- if (std::distance(begin, end) < 2) {
- return begin;
- }
-
- static_assert(std::is_integral<decltype(getWeight(*begin))>::value);
-
- std::vector<std::pair<const ui64, const Iterator>> borders;
- borders.reserve(std::distance(begin, end));
-
- ui64 total = 0;
- for (auto it = begin; it != end; ++it) {
- total += getWeight(*it);
- borders.emplace_back(total, it);
- }
-
+
+namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering {
+
+template <typename Iterator, typename GetWeightFunc>
+Iterator SelectByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) {
+ if (std::distance(begin, end) < 2) {
+ return begin;
+ }
+
+ static_assert(std::is_integral<decltype(getWeight(*begin))>::value);
+
+ std::vector<std::pair<const ui64, const Iterator>> borders;
+ borders.reserve(std::distance(begin, end));
+
+ ui64 total = 0;
+ for (auto it = begin; it != end; ++it) {
+ total += getWeight(*it);
+ borders.emplace_back(total, it);
+ }
+
Y_VERIFY(total);
- const ui64 borderToFind = hashValue % total + 1;
-
- const auto bordersIt = lower_bound(borders.begin(), borders.end(), borderToFind,
- [](const auto& borderInfo, const ui64& toFind) { return borderInfo.first < toFind; });
-
- return bordersIt->second;
-}
-
-template <typename Iterator, typename GetWeightFunc>
-void OrderByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) {
- auto currentBegin = begin;
- while (std::distance(currentBegin, end) > 1) {
- auto selectedIt = SelectByHashAndWeight(currentBegin, end, hashValue, getWeight);
- if (currentBegin != selectedIt) {
- std::swap(*currentBegin, *selectedIt);
- }
- ++currentBegin;
- }
-}
-
-} // namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering
+ const ui64 borderToFind = hashValue % total + 1;
+
+ const auto bordersIt = lower_bound(borders.begin(), borders.end(), borderToFind,
+ [](const auto& borderInfo, const ui64& toFind) { return borderInfo.first < toFind; });
+
+ return bordersIt->second;
+}
+
+template <typename Iterator, typename GetWeightFunc>
+void OrderByHashAndWeight(Iterator begin, Iterator end, const ui64 hashValue, const GetWeightFunc getWeight) {
+ auto currentBegin = begin;
+ while (std::distance(currentBegin, end) > 1) {
+ auto selectedIt = SelectByHashAndWeight(currentBegin, end, hashValue, getWeight);
+ if (currentBegin != selectedIt) {
+ std::swap(*currentBegin, *selectedIt);
+ }
+ ++currentBegin;
+ }
+}
+
+} // namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp
index b56071421a..068f90e293 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp
+++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering_ut.cpp
@@ -1,55 +1,55 @@
#include <ydb/services/persqueue_cluster_discovery/cluster_ordering/weighed_ordering.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/testing/unittest/tests_data.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/string.h>
-
-using namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering;
-
-Y_UNIT_TEST_SUITE(TWeighedOrderingTest) {
- struct TCluster {
- TString Name;
- ui64 Weight = 0;
-
- bool operator==(const TCluster& other) const {
- return Name == other.Name && Weight == other.Weight;
- }
- };
-
- Y_UNIT_TEST(SimpleSelectionTest) {
- std::vector<TCluster> clusters = {{"sas", 1}, {"man", 1}};
-
- UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 0, [](const TCluster& c){ return c.Weight; })->Name, "sas");
- UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 1, [](const TCluster& c){ return c.Weight; })->Name, "man");
- UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100500, [](const TCluster& c){ return c.Weight; })->Name, "sas");
- UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; })->Name, "man");
- }
-
- Y_UNIT_TEST(WeighedSelectionTest) {
- std::vector<TCluster> clusters = {{"sas", 10}, {"man", 10000}};
-
- THashMap<TString, ui64> counters;
- for (ui64 hashValue = 0; hashValue < 10000000; hashValue += 661) {
- ++counters[SelectByHashAndWeight(begin(clusters), end(clusters), hashValue, [](const TCluster& c){ return c.Weight; })->Name];
- }
-
- UNIT_ASSERT_VALUES_EQUAL(counters.size(), 2);
- UNIT_ASSERT(counters["man"] > counters["sas"] * 1000); // ensure the imbalance
- }
-
- Y_UNIT_TEST(WeighedOrderingTest) {
- std::vector<TCluster> clusters = {{"iva", 1}, {"man", 2}, {"myt", 3}, {"sas", 4}, {"vla", 5}};
-
- auto firstOrder = clusters;
- OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; });
- auto secondOrder = clusters;
- OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; });
- auto thirdOrder = clusters;
-
- UNIT_ASSERT(firstOrder != secondOrder);
- UNIT_ASSERT(secondOrder != thirdOrder);
- UNIT_ASSERT(firstOrder != thirdOrder);
- }
-}
+
+#include <util/generic/hash.h>
+#include <util/generic/string.h>
+
+using namespace NKikimr::NPQ::NClusterDiscovery::NClusterOrdering;
+
+Y_UNIT_TEST_SUITE(TWeighedOrderingTest) {
+ struct TCluster {
+ TString Name;
+ ui64 Weight = 0;
+
+ bool operator==(const TCluster& other) const {
+ return Name == other.Name && Weight == other.Weight;
+ }
+ };
+
+ Y_UNIT_TEST(SimpleSelectionTest) {
+ std::vector<TCluster> clusters = {{"sas", 1}, {"man", 1}};
+
+ UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 0, [](const TCluster& c){ return c.Weight; })->Name, "sas");
+ UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 1, [](const TCluster& c){ return c.Weight; })->Name, "man");
+ UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100500, [](const TCluster& c){ return c.Weight; })->Name, "sas");
+ UNIT_ASSERT_STRINGS_EQUAL(SelectByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; })->Name, "man");
+ }
+
+ Y_UNIT_TEST(WeighedSelectionTest) {
+ std::vector<TCluster> clusters = {{"sas", 10}, {"man", 10000}};
+
+ THashMap<TString, ui64> counters;
+ for (ui64 hashValue = 0; hashValue < 10000000; hashValue += 661) {
+ ++counters[SelectByHashAndWeight(begin(clusters), end(clusters), hashValue, [](const TCluster& c){ return c.Weight; })->Name];
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(counters.size(), 2);
+ UNIT_ASSERT(counters["man"] > counters["sas"] * 1000); // ensure the imbalance
+ }
+
+ Y_UNIT_TEST(WeighedOrderingTest) {
+ std::vector<TCluster> clusters = {{"iva", 1}, {"man", 2}, {"myt", 3}, {"sas", 4}, {"vla", 5}};
+
+ auto firstOrder = clusters;
+ OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; });
+ auto secondOrder = clusters;
+ OrderByHashAndWeight(begin(clusters), end(clusters), 100501, [](const TCluster& c){ return c.Weight; });
+ auto thirdOrder = clusters;
+
+ UNIT_ASSERT(firstOrder != secondOrder);
+ UNIT_ASSERT(secondOrder != thirdOrder);
+ UNIT_ASSERT(firstOrder != thirdOrder);
+ }
+}
diff --git a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make
index f01a54c355..6978da1d9b 100644
--- a/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/cluster_ordering/ya.make
@@ -1,13 +1,13 @@
-OWNER(
- radix
+OWNER(
+ radix
g:kikimr
g:logbroker
-)
-
-LIBRARY()
-
-SRCS(
- weighed_ordering.cpp
-)
-
-END()
+)
+
+LIBRARY()
+
+SRCS(
+ weighed_ordering.cpp
+)
+
+END()
diff --git a/ydb/services/persqueue_cluster_discovery/counters.cpp b/ydb/services/persqueue_cluster_discovery/counters.cpp
index 3cfc4eb863..c1a9bf7589 100644
--- a/ydb/services/persqueue_cluster_discovery/counters.cpp
+++ b/ydb/services/persqueue_cluster_discovery/counters.cpp
@@ -1,56 +1,56 @@
-#include "counters.h"
-
-namespace NKikimr::NPQ::NClusterDiscovery::NCounters {
-
-#define SETUP_SIMPLE_COUNTER(name, derived) \
- name(counters->GetCounter(#name, derived))
-
-TClusterDiscoveryCounters::TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters,
- NClusterTracker::TClustersList::TConstPtr clustersList,
- TLabeledAddressClassifier::TConstPtr addressClassifier)
- : SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false)
- , SETUP_SIMPLE_COUNTER(ClustersListUpdateLagSeconds, false)
- , SETUP_SIMPLE_COUNTER(HealthProbe, false)
- , SETUP_SIMPLE_COUNTER(DroppedRequestsCount, true)
- , SETUP_SIMPLE_COUNTER(TotalRequestsCount, true)
- , SETUP_SIMPLE_COUNTER(SuccessfulRequestsCount, true)
- , SETUP_SIMPLE_COUNTER(BadRequestsCount, true)
- , SETUP_SIMPLE_COUNTER(FailedRequestsCount, true)
- , SETUP_SIMPLE_COUNTER(WriteDiscoveriesCount, true)
- , SETUP_SIMPLE_COUNTER(ReadDiscoveriesCount, true)
- , SETUP_SIMPLE_COUNTER(InfracloudRequestsCount, true)
-{
- DiscoveryWorkingDurationMs =
- counters->GetHistogram("DiscoveryWorkingDurationMs",
- NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 25, 50, 100, 500}));
-
- if (clustersList) {
- for (const auto& cluster : clustersList->Clusters) {
- TimesPrioritizedForWrite[cluster.Name] =
- counters->GetSubgroup("prioritized_cluster", cluster.Name)->GetCounter("TimesPrioritizedForWrite", true);
- }
- }
-
- if (addressClassifier) {
- for (const auto& label : addressClassifier->GetLabels()) {
- TimesResolvedByClassifier[label] =
- counters->GetSubgroup("resolved_dc", label)->GetCounter("TimesResolvedByClassifier", true);
- }
- }
-
- for (
- std::underlying_type<WriteSessionClusters::SelectionReason>::type reason = WriteSessionClusters::SELECTION_REASON_UNSPECIFIED;
- reason <= WriteSessionClusters::CONSISTENT_DISTRIBUTION;
- ++reason
- )
- {
- const auto currentReason = static_cast<WriteSessionClusters::SelectionReason>(reason);
- PrimaryClusterSelectionReasons[currentReason] =
- counters->GetSubgroup("primary_cluster_selection_reason", ToString(currentReason))->GetCounter("TimesMentionedForWrite", true);
- }
-
-}
-
-#undef SETUP_SIMPLE_COUNTER
-
-} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters
+#include "counters.h"
+
+namespace NKikimr::NPQ::NClusterDiscovery::NCounters {
+
+#define SETUP_SIMPLE_COUNTER(name, derived) \
+ name(counters->GetCounter(#name, derived))
+
+TClusterDiscoveryCounters::TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters,
+ NClusterTracker::TClustersList::TConstPtr clustersList,
+ TLabeledAddressClassifier::TConstPtr addressClassifier)
+ : SETUP_SIMPLE_COUNTER(NetDataUpdateLagSeconds, false)
+ , SETUP_SIMPLE_COUNTER(ClustersListUpdateLagSeconds, false)
+ , SETUP_SIMPLE_COUNTER(HealthProbe, false)
+ , SETUP_SIMPLE_COUNTER(DroppedRequestsCount, true)
+ , SETUP_SIMPLE_COUNTER(TotalRequestsCount, true)
+ , SETUP_SIMPLE_COUNTER(SuccessfulRequestsCount, true)
+ , SETUP_SIMPLE_COUNTER(BadRequestsCount, true)
+ , SETUP_SIMPLE_COUNTER(FailedRequestsCount, true)
+ , SETUP_SIMPLE_COUNTER(WriteDiscoveriesCount, true)
+ , SETUP_SIMPLE_COUNTER(ReadDiscoveriesCount, true)
+ , SETUP_SIMPLE_COUNTER(InfracloudRequestsCount, true)
+{
+ DiscoveryWorkingDurationMs =
+ counters->GetHistogram("DiscoveryWorkingDurationMs",
+ NMonitoring::ExplicitHistogram({0, 1, 2, 5, 10, 25, 50, 100, 500}));
+
+ if (clustersList) {
+ for (const auto& cluster : clustersList->Clusters) {
+ TimesPrioritizedForWrite[cluster.Name] =
+ counters->GetSubgroup("prioritized_cluster", cluster.Name)->GetCounter("TimesPrioritizedForWrite", true);
+ }
+ }
+
+ if (addressClassifier) {
+ for (const auto& label : addressClassifier->GetLabels()) {
+ TimesResolvedByClassifier[label] =
+ counters->GetSubgroup("resolved_dc", label)->GetCounter("TimesResolvedByClassifier", true);
+ }
+ }
+
+ for (
+ std::underlying_type<WriteSessionClusters::SelectionReason>::type reason = WriteSessionClusters::SELECTION_REASON_UNSPECIFIED;
+ reason <= WriteSessionClusters::CONSISTENT_DISTRIBUTION;
+ ++reason
+ )
+ {
+ const auto currentReason = static_cast<WriteSessionClusters::SelectionReason>(reason);
+ PrimaryClusterSelectionReasons[currentReason] =
+ counters->GetSubgroup("primary_cluster_selection_reason", ToString(currentReason))->GetCounter("TimesMentionedForWrite", true);
+ }
+
+}
+
+#undef SETUP_SIMPLE_COUNTER
+
+} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters
diff --git a/ydb/services/persqueue_cluster_discovery/counters.h b/ydb/services/persqueue_cluster_discovery/counters.h
index fcfbd8db65..495b2e6939 100644
--- a/ydb/services/persqueue_cluster_discovery/counters.h
+++ b/ydb/services/persqueue_cluster_discovery/counters.h
@@ -1,53 +1,53 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/grpc_services/grpc_request_proxy.h>
-
+
#include <ydb/core/persqueue/cluster_tracker.h>
#include <ydb/core/util/address_classifier.h>
-
+
#include <library/cpp/monlib/dynamic_counters/counters.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/ptr.h>
-#include <util/generic/noncopyable.h>
-
-namespace NKikimr::NPQ::NClusterDiscovery::NCounters {
-
-using namespace NAddressClassifier;
-using namespace NClusterTracker;
-using namespace NMonitoring;
-using namespace Ydb::PersQueue::ClusterDiscovery;
-
-struct TClusterDiscoveryCounters : TAtomicRefCount<TClusterDiscoveryCounters>, TNonCopyable {
- using TPtr = TIntrusivePtr<TClusterDiscoveryCounters>;
- using TCounterPtr = TDynamicCounters::TCounterPtr;
-
- TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters,
- TClustersList::TConstPtr clustersList,
- TLabeledAddressClassifier::TConstPtr addressClassifier);
-
- TCounterPtr NetDataUpdateLagSeconds; // Seconds since last net data update
- TCounterPtr ClustersListUpdateLagSeconds; // Seconds since last clusters list update
- TCounterPtr HealthProbe; // Health check results timeline
-
- TCounterPtr DroppedRequestsCount; // GRpc status: UNAVAILABLE
- TCounterPtr TotalRequestsCount; // GRpc status: ANY
- TCounterPtr SuccessfulRequestsCount; // GRpc status: SUCCESS
- TCounterPtr BadRequestsCount; // GRrpc status: BAD_REQUEST
- TCounterPtr FailedRequestsCount; // GRpc status: INTERNAL_ERROR
-
- TCounterPtr WriteDiscoveriesCount; // Write discovery proto elements count in request
- TCounterPtr ReadDiscoveriesCount; // Read discovery proto elements count in request
-
- TCounterPtr InfracloudRequestsCount; // Requests coming from infracloud clients
-
- THistogramPtr DiscoveryWorkingDurationMs; // NB: dropped requests are not measured
-
- THashMap<TString, TCounterPtr> TimesPrioritizedForWrite; // Times a cluster with a certain name was prioritized
- THashMap<TString, TCounterPtr> TimesResolvedByClassifier; // Times a client's datacenter with a certain name was resolved
-
- // Primary cluster selection reasons for write sessions
- THashMap<WriteSessionClusters::SelectionReason, TCounterPtr> PrimaryClusterSelectionReasons;
-};
-
-} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters
+
+#include <util/generic/hash.h>
+#include <util/generic/ptr.h>
+#include <util/generic/noncopyable.h>
+
+namespace NKikimr::NPQ::NClusterDiscovery::NCounters {
+
+using namespace NAddressClassifier;
+using namespace NClusterTracker;
+using namespace NMonitoring;
+using namespace Ydb::PersQueue::ClusterDiscovery;
+
+struct TClusterDiscoveryCounters : TAtomicRefCount<TClusterDiscoveryCounters>, TNonCopyable {
+ using TPtr = TIntrusivePtr<TClusterDiscoveryCounters>;
+ using TCounterPtr = TDynamicCounters::TCounterPtr;
+
+ TClusterDiscoveryCounters(TIntrusivePtr<TDynamicCounters> counters,
+ TClustersList::TConstPtr clustersList,
+ TLabeledAddressClassifier::TConstPtr addressClassifier);
+
+ TCounterPtr NetDataUpdateLagSeconds; // Seconds since last net data update
+ TCounterPtr ClustersListUpdateLagSeconds; // Seconds since last clusters list update
+ TCounterPtr HealthProbe; // Health check results timeline
+
+ TCounterPtr DroppedRequestsCount; // GRpc status: UNAVAILABLE
+ TCounterPtr TotalRequestsCount; // GRpc status: ANY
+ TCounterPtr SuccessfulRequestsCount; // GRpc status: SUCCESS
+ TCounterPtr BadRequestsCount; // GRrpc status: BAD_REQUEST
+ TCounterPtr FailedRequestsCount; // GRpc status: INTERNAL_ERROR
+
+ TCounterPtr WriteDiscoveriesCount; // Write discovery proto elements count in request
+ TCounterPtr ReadDiscoveriesCount; // Read discovery proto elements count in request
+
+ TCounterPtr InfracloudRequestsCount; // Requests coming from infracloud clients
+
+ THistogramPtr DiscoveryWorkingDurationMs; // NB: dropped requests are not measured
+
+ THashMap<TString, TCounterPtr> TimesPrioritizedForWrite; // Times a cluster with a certain name was prioritized
+ THashMap<TString, TCounterPtr> TimesResolvedByClassifier; // Times a client's datacenter with a certain name was resolved
+
+ // Primary cluster selection reasons for write sessions
+ THashMap<WriteSessionClusters::SelectionReason, TCounterPtr> PrimaryClusterSelectionReasons;
+};
+
+} // namespace NKikimr::NPQ::NClusterDiscovery::NCounters
diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp
index 0a729b4d09..086d162443 100644
--- a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp
+++ b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp
@@ -1,86 +1,86 @@
-#include "grpc_service.h"
-#include "cluster_discovery_service.h"
-
+#include "grpc_service.h"
+#include "cluster_discovery_service.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/counters.h>
-
+
#include <ydb/core/grpc_services/grpc_helper.h>
#include <ydb/core/grpc_services/grpc_request_proxy.h>
#include <ydb/core/grpc_services/rpc_calls.h>
-
+
#include <library/cpp/grpc/server/grpc_request.h>
-
-
-namespace NKikimr::NGRpcService {
-
+
+
+namespace NKikimr::NGRpcService {
+
TGRpcPQClusterDiscoveryService::TGRpcPQClusterDiscoveryService(
NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id,
const TMaybe<ui64>& requestsInflightLimit
)
- : ActorSystem_(system)
- , Counters_(counters)
- , GRpcRequestProxyId_(id)
+ : ActorSystem_(system)
+ , Counters_(counters)
+ , GRpcRequestProxyId_(id)
{
if (requestsInflightLimit.Defined()) {
Limiter = MakeHolder<NGrpc::TGlobalLimiter>(requestsInflightLimit.GetRef());
}
}
-
+
void TGRpcPQClusterDiscoveryService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) {
- CQ_ = cq;
-
- if (ActorSystem_->AppData<TAppData>()->PQClusterDiscoveryConfig.GetEnabled()) {
- IActor* actor = NPQ::NClusterDiscovery::CreateClusterDiscoveryService(GetServiceCounters(Counters_, "persqueue")->GetSubgroup("subsystem", "cluster_discovery"));
+ CQ_ = cq;
+
+ if (ActorSystem_->AppData<TAppData>()->PQClusterDiscoveryConfig.GetEnabled()) {
+ IActor* actor = NPQ::NClusterDiscovery::CreateClusterDiscoveryService(GetServiceCounters(Counters_, "persqueue")->GetSubgroup("subsystem", "cluster_discovery"));
TActorId clusterDiscoveryServiceId = ActorSystem_->Register(actor, TMailboxType::HTSwap, ActorSystem_->AppData<TAppData>()->UserPoolId);
- ActorSystem_->RegisterLocalService(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID(), clusterDiscoveryServiceId);
-
+ ActorSystem_->RegisterLocalService(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID(), clusterDiscoveryServiceId);
+
SetupIncomingRequests(std::move(logger));
- }
-}
-
+ }
+}
+
void TGRpcPQClusterDiscoveryService::SetGlobalLimiterHandle(NGrpc::TGlobalLimiter*) {
-}
-
-bool TGRpcPQClusterDiscoveryService::IncRequest() {
+}
+
+bool TGRpcPQClusterDiscoveryService::IncRequest() {
if (Limiter) {
return Limiter->Inc();
}
return true;
-}
-
-void TGRpcPQClusterDiscoveryService::DecRequest() {
+}
+
+void TGRpcPQClusterDiscoveryService::DecRequest() {
if (Limiter) {
Limiter->Dec();
}
-}
-
+}
+
void TGRpcPQClusterDiscoveryService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
- auto getCounterBlock = NGRpcService::CreateCounterCb(Counters_, ActorSystem_);
-
-#ifdef ADD_REQUEST
-#error ADD_REQUEST macro already defined
-#endif
-#define ADD_REQUEST(NAME, IN, OUT, ACTION) \
- MakeIntrusive<TGRpcRequest<Ydb::PersQueue::ClusterDiscovery::IN, Ydb::PersQueue::ClusterDiscovery::OUT, TGRpcPQClusterDiscoveryService>>(this, &Service_, CQ_, \
+ auto getCounterBlock = NGRpcService::CreateCounterCb(Counters_, ActorSystem_);
+
+#ifdef ADD_REQUEST
+#error ADD_REQUEST macro already defined
+#endif
+#define ADD_REQUEST(NAME, IN, OUT, ACTION) \
+ MakeIntrusive<TGRpcRequest<Ydb::PersQueue::ClusterDiscovery::IN, Ydb::PersQueue::ClusterDiscovery::OUT, TGRpcPQClusterDiscoveryService>>(this, &Service_, CQ_, \
[this](NGrpc::IRequestContextBase* ctx) { \
- NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \
- ACTION; \
- }, &Ydb::PersQueue::V1::ClusterDiscoveryService::AsyncService::Request ## NAME, \
+ NGRpcService::ReportGrpcReqToMon(*ActorSystem_, ctx->GetPeer()); \
+ ACTION; \
+ }, &Ydb::PersQueue::V1::ClusterDiscoveryService::AsyncService::Request ## NAME, \
#NAME, logger, getCounterBlock("pq_cluster_discovery", #NAME))->Run();
-
- ADD_REQUEST(DiscoverClusters, DiscoverClustersRequest, DiscoverClustersResponse, {
- ActorSystem_->Send(GRpcRequestProxyId_, new TEvDiscoverPQClustersRequest(ctx));
- })
-#undef ADD_REQUEST
-
-}
-
-void TGRpcPQClusterDiscoveryService::StopService() noexcept {
- TGrpcServiceBase::StopService();
-}
-
-void TGRpcRequestProxy::Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx) {
- ctx.Send(ev->Forward(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID()));
-}
-
-} // namespace NKikimr::NGRpcService
+
+ ADD_REQUEST(DiscoverClusters, DiscoverClustersRequest, DiscoverClustersResponse, {
+ ActorSystem_->Send(GRpcRequestProxyId_, new TEvDiscoverPQClustersRequest(ctx));
+ })
+#undef ADD_REQUEST
+
+}
+
+void TGRpcPQClusterDiscoveryService::StopService() noexcept {
+ TGrpcServiceBase::StopService();
+}
+
+void TGRpcRequestProxy::Handle(TEvDiscoverPQClustersRequest::TPtr& ev, const TActorContext& ctx) {
+ ctx.Send(ev->Forward(NPQ::NClusterDiscovery::MakeClusterDiscoveryServiceID()));
+}
+
+} // namespace NKikimr::NGRpcService
diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.h b/ydb/services/persqueue_cluster_discovery/grpc_service.h
index 5d430364b8..7bc17ba2f5 100644
--- a/ydb/services/persqueue_cluster_discovery/grpc_service.h
+++ b/ydb/services/persqueue_cluster_discovery/grpc_service.h
@@ -1,33 +1,33 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/actors/core/actorsystem.h>
#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h>
#include <library/cpp/grpc/server/grpc_server.h>
-
-namespace NKikimr::NGRpcService {
-
-class TGRpcPQClusterDiscoveryService
+
+namespace NKikimr::NGRpcService {
+
+class TGRpcPQClusterDiscoveryService
: public NGrpc::TGrpcServiceBase<Ydb::PersQueue::V1::ClusterDiscoveryService> {
-public:
+public:
TGRpcPQClusterDiscoveryService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
NActors::TActorId id, const TMaybe<ui64>& requestsInflightLimit = Nothing());
-
+
void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override;
void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override;
- bool IncRequest();
- void DecRequest();
- void StopService() noexcept override;
-
+ bool IncRequest();
+ void DecRequest();
+ void StopService() noexcept override;
+
using NGrpc::TGrpcServiceBase<Ydb::PersQueue::V1::ClusterDiscoveryService>::GetService;
-private:
+private:
void SetupIncomingRequests(NGrpc::TLoggerPtr logger);
-
- NActors::TActorSystem* ActorSystem_;
+
+ NActors::TActorSystem* ActorSystem_;
grpc::ServerCompletionQueue* CQ_ = nullptr;
-
- TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_;
+
+ TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_;
NActors::TActorId GRpcRequestProxyId_;
THolder<NGrpc::TGlobalLimiter> Limiter;
-};
-
-}
+};
+
+}
diff --git a/ydb/services/persqueue_cluster_discovery/ut/ya.make b/ydb/services/persqueue_cluster_discovery/ut/ya.make
index 1ef703a01f..352988d2fe 100644
--- a/ydb/services/persqueue_cluster_discovery/ut/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/ut/ya.make
@@ -1,34 +1,34 @@
UNITTEST_FOR(ydb/services/persqueue_cluster_discovery)
-
+
OWNER(
radix
g:kikimr
g:logbroker
)
-
-FORK_SUBTESTS()
-IF (WITH_VALGRIND)
- TIMEOUT(1800)
- SIZE(LARGE)
- TAG(ya:fat)
+FORK_SUBTESTS()
+
+IF (WITH_VALGRIND)
+ TIMEOUT(1800)
+ SIZE(LARGE)
+ TAG(ya:fat)
REQUIREMENTS(ram:32)
-ELSE()
- TIMEOUT(600)
- SIZE(MEDIUM)
-ENDIF()
-
-SRCS(
- cluster_discovery_service_ut.cpp
-)
-
-PEERDIR(
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ cluster_discovery_service_ut.cpp
+)
+
+PEERDIR(
library/cpp/actors/http
ydb/core/testlib
ydb/public/api/grpc
ydb/services/persqueue_cluster_discovery
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/services/persqueue_cluster_discovery/ya.make b/ydb/services/persqueue_cluster_discovery/ya.make
index aea0ea543e..23917ae376 100644
--- a/ydb/services/persqueue_cluster_discovery/ya.make
+++ b/ydb/services/persqueue_cluster_discovery/ya.make
@@ -1,19 +1,19 @@
-LIBRARY()
-
-OWNER(
- radix
+LIBRARY()
+
+OWNER(
+ radix
g:kikimr
g:logbroker
-)
-
-SRCS(
- cluster_discovery_service.cpp
- cluster_discovery_worker.cpp
- counters.cpp
- grpc_service.cpp
-)
-
-PEERDIR(
+)
+
+SRCS(
+ cluster_discovery_service.cpp
+ cluster_discovery_worker.cpp
+ counters.cpp
+ grpc_service.cpp
+)
+
+PEERDIR(
ydb/core/base
ydb/core/client/server
ydb/core/grpc_services
@@ -26,9 +26,9 @@ PEERDIR(
ydb/public/api/grpc/draft
ydb/public/api/protos
ydb/services/persqueue_cluster_discovery/cluster_ordering
-)
-
-END()
+)
+
+END()
RECURSE_FOR_TESTS(
ut
diff --git a/ydb/services/persqueue_v1/grpc_pq_read.cpp b/ydb/services/persqueue_v1/grpc_pq_read.cpp
index eb7fbd35b1..6b7e72fdd0 100644
--- a/ydb/services/persqueue_v1/grpc_pq_read.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_read.cpp
@@ -4,8 +4,8 @@
#include <ydb/core/grpc_services/grpc_helper.h>
#include <ydb/core/tx/scheme_board/cache.h>
-#include <algorithm>
-
+#include <algorithm>
+
using namespace NActors;
using namespace NKikimrClient;
@@ -71,24 +71,24 @@ void TPQReadService::Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpdat
DatacenterClassifier = ev->Get()->Classifier;
}
-void TPQReadService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
- Y_VERIFY(ev->Get()->ClustersList);
-
- Y_VERIFY(ev->Get()->ClustersList->Clusters.size());
-
- const auto& clusters = ev->Get()->ClustersList->Clusters;
-
- LocalCluster = {};
-
- auto it = std::find_if(begin(clusters), end(clusters), [](const auto& cluster) { return cluster.IsLocal; });
- if (it != end(clusters)) {
- LocalCluster = it->Name;
- }
-
- Clusters.resize(clusters.size());
- for (size_t i = 0; i < clusters.size(); ++i) {
- Clusters[i] = clusters[i].Name;
- }
+void TPQReadService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev) {
+ Y_VERIFY(ev->Get()->ClustersList);
+
+ Y_VERIFY(ev->Get()->ClustersList->Clusters.size());
+
+ const auto& clusters = ev->Get()->ClustersList->Clusters;
+
+ LocalCluster = {};
+
+ auto it = std::find_if(begin(clusters), end(clusters), [](const auto& cluster) { return cluster.IsLocal; });
+ if (it != end(clusters)) {
+ LocalCluster = it->Name;
+ }
+
+ Clusters.resize(clusters.size());
+ for (size_t i = 0; i < clusters.size(); ++i) {
+ Clusters[i] = clusters[i].Name;
+ }
TopicsHandler->UpdateClusters(Clusters, LocalCluster);
}
diff --git a/ydb/services/persqueue_v1/grpc_pq_read.h b/ydb/services/persqueue_v1/grpc_pq_read.h
index 3348688a0a..558063ed13 100644
--- a/ydb/services/persqueue_v1/grpc_pq_read.h
+++ b/ydb/services/persqueue_v1/grpc_pq_read.h
@@ -46,7 +46,7 @@ private:
switch (ev->GetTypeRewrite()) {
HFunc(NKikimr::NGRpcService::TEvStreamPQReadRequest, Handle);
HFunc(NKikimr::NGRpcService::TEvPQReadInfoRequest, Handle);
- hFunc(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate, Handle);
+ hFunc(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate, Handle);
HFunc(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate, Handle);
HFunc(TEvPQProxy::TEvSessionDead, Handle);
}
@@ -55,7 +55,7 @@ private:
private:
void Handle(NKikimr::NGRpcService::TEvStreamPQReadRequest::TPtr& ev, const TActorContext& ctx);
void Handle(NKikimr::NGRpcService::TEvPQReadInfoRequest::TPtr& ev, const TActorContext& ctx);
- void Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev);
+ void Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev);
void Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpdate::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPQProxy::TEvSessionDead::TPtr& ev, const TActorContext& ctx);
diff --git a/ydb/services/persqueue_v1/grpc_pq_write.cpp b/ydb/services/persqueue_v1/grpc_pq_write.cpp
index c18175de4e..5f17a2f3ab 100644
--- a/ydb/services/persqueue_v1/grpc_pq_write.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_write.cpp
@@ -66,14 +66,14 @@ void TPQWriteService::Handle(NNetClassifier::TEvNetClassifier::TEvClassifierUpda
void TPQWriteService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClustersUpdate::TPtr& ev, const TActorContext& ctx) {
- Y_VERIFY(ev->Get()->ClustersList);
- Y_VERIFY(ev->Get()->ClustersList->Clusters.size());
-
- const auto& clusters = ev->Get()->ClustersList->Clusters;
-
- LocalCluster = "";
- Enabled = false;
-
+ Y_VERIFY(ev->Get()->ClustersList);
+ Y_VERIFY(ev->Get()->ClustersList->Clusters.size());
+
+ const auto& clusters = ev->Get()->ClustersList->Clusters;
+
+ LocalCluster = "";
+ Enabled = false;
+
// Rebalance load on installation clusters: if preferred cluster is enabled and session is alive long enough close it so client can recreate it in preferred cluster
auto remoteClusterEnabledDelay = TDuration::Seconds(AppData(ctx)->PQConfig.GetRemoteClusterEnabledDelaySec());
auto closeClientSessionWithEnabledRemotePreferredClusterDelay = TDuration::Seconds(AppData(ctx)->PQConfig.GetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec());
@@ -86,7 +86,7 @@ void TPQWriteService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvCluster
Enabled = cluster.IsEnabled;
continue;
}
-
+
remoteClusters.emplace(cluster.Name);
if (!cluster.IsEnabled) {
@@ -126,7 +126,7 @@ void TPQWriteService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvCluster
const auto& workerID = Sessions[session.first];
Send(workerID, new TEvPQProxy::TEvDieCommand(closeReason, PersQueue::ErrorCode::PREFERRED_CLUSTER_MISMATCHED));
}
- }
+ }
}
}
}
@@ -180,7 +180,7 @@ void TPQWriteService::Handle(NKikimr::NGRpcService::TEvStreamPQWriteRequest::TPt
TString localCluster = AvailableLocalCluster(ctx);
if (HaveClusters && localCluster.empty()) {
ev->Get()->GetStreamCtx()->Attach(ctx.SelfID);
- if (LocalCluster) {
+ if (LocalCluster) {
LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "new grpc connection failed - cluster disabled");
ev->Get()->GetStreamCtx()->WriteAndFinish(FillWriteResponse("cluster disabled", PersQueue::ErrorCode::CLUSTER_DISABLED), grpc::Status::OK); //CANCELLED
} else {
diff --git a/ydb/services/persqueue_v1/grpc_pq_write.h b/ydb/services/persqueue_v1/grpc_pq_write.h
index 787c47e262..73148388b2 100644
--- a/ydb/services/persqueue_v1/grpc_pq_write.h
+++ b/ydb/services/persqueue_v1/grpc_pq_write.h
@@ -10,7 +10,7 @@
#include <library/cpp/actors/core/actorsystem.h>
#include <util/generic/hash.h>
-#include <util/generic/maybe.h>
+#include <util/generic/maybe.h>
#include <util/system/mutex.h>
namespace NKikimr {
@@ -74,7 +74,7 @@ private:
TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
ui32 MaxSessions;
- TMaybe<TString> LocalCluster;
+ TMaybe<TString> LocalCluster;
bool Enabled;
TString SelectSourceIdQuery;
TString UpdateSourceIdQuery;
diff --git a/ydb/tests/functional/sqs/sqs_requests_client.py b/ydb/tests/functional/sqs/sqs_requests_client.py
index e7823ea22e..6b7545503f 100644
--- a/ydb/tests/functional/sqs/sqs_requests_client.py
+++ b/ydb/tests/functional/sqs/sqs_requests_client.py
@@ -28,23 +28,23 @@ def to_bytes(v):
return v.encode('utf-8')
-def auth_string(user):
+def auth_string(user):
return "AWS4-HMAC-SHA256 Credential={user}/{date}/yandex/sqs/aws4_request".format(
- user=user, date=DEFAULT_DATE
+ user=user, date=DEFAULT_DATE
)
-def auth_headers(user, security_token=None, iam_token=None):
- headers = {
- 'X-Amz-Date': '{}T104419Z'.format(DEFAULT_DATE),
- 'Authorization': auth_string(user)
+def auth_headers(user, security_token=None, iam_token=None):
+ headers = {
+ 'X-Amz-Date': '{}T104419Z'.format(DEFAULT_DATE),
+ 'Authorization': auth_string(user)
}
- if security_token is not None:
- headers['X-Amz-Security-Token'] = security_token
- if iam_token is not None:
- headers['X-YaCloud-SubjectToken'] = iam_token
-
- return headers
+ if security_token is not None:
+ headers['X-Amz-Security-Token'] = security_token
+ if iam_token is not None:
+ headers['X-YaCloud-SubjectToken'] = iam_token
+
+ return headers
SQS_ATTRIBUTE_TYPES = {'String': 'StringValue', 'Number': 'StringValue', 'Binary': 'BinaryValue'}
@@ -79,38 +79,38 @@ class SqsChangeMessageVisibilityParams(object):
class SqsHttpApi(object):
- def __init__(self, server, port, user, raise_on_error=False, timeout=REQUEST_TIMEOUT, security_token=None, force_private=False, iam_token=None, folder_id=None):
- self.__auth_headers = auth_headers(user, security_token, iam_token)
+ def __init__(self, server, port, user, raise_on_error=False, timeout=REQUEST_TIMEOUT, security_token=None, force_private=False, iam_token=None, folder_id=None):
+ self.__auth_headers = auth_headers(user, security_token, iam_token)
if not server.startswith('http'):
server = 'http://' + server
self.__request = requests.Request(
'POST',
"{}:{}".format(server, port),
- headers=auth_headers(user, security_token, iam_token)
+ headers=auth_headers(user, security_token, iam_token)
)
self.__private_request = requests.Request(
'POST',
"{}:{}/private".format(server, port),
- headers=auth_headers(user, security_token, iam_token)
+ headers=auth_headers(user, security_token, iam_token)
)
self.__session = requests.Session()
self.__raise_on_error = raise_on_error
self.__user = user
self.__timeout = timeout
- self.__security_token = security_token
- assert(isinstance(force_private, bool))
- self.__force_private = force_private
- self.__folder_id = folder_id
+ self.__security_token = security_token
+ assert(isinstance(force_private, bool))
+ self.__force_private = force_private
+ self.__folder_id = folder_id
def execute_request(self, action, extract_result_method, default=None, private=False, **params):
if params is None:
params = {}
params['Action'] = action
-
- if self.__folder_id is not None:
- params['folderId'] = self.__folder_id
-
- if self.__force_private or private:
+
+ if self.__folder_id is not None:
+ params['folderId'] = self.__folder_id
+
+ if self.__force_private or private:
request = self.__private_request
else:
request = self.__request
@@ -141,7 +141,7 @@ class SqsHttpApi(object):
response.status_code, response.reason, response.text
))
# Assert that no internal info will be given to user
- assert response.text.find('.cpp:') == -1, 'No internal info should be given to user'
+ assert response.text.find('.cpp:') == -1, 'No internal info should be given to user'
if self.__raise_on_error:
raise RuntimeError(
"Request {} failed with status {} and text {}".format(
@@ -165,13 +165,13 @@ class SqsHttpApi(object):
UserName=username
)
- def delete_user(self, username):
- return self.execute_request(
- action='DeleteUser',
- extract_result_method=lambda x: x['DeleteUserResponse']['ResponseMetadata'],
- UserName=username
- )
-
+ def delete_user(self, username):
+ return self.execute_request(
+ action='DeleteUser',
+ extract_result_method=lambda x: x['DeleteUserResponse']['ResponseMetadata'],
+ UserName=username
+ )
+
def list_users(self):
return self.execute_request(
action='ListUsers',
@@ -191,13 +191,13 @@ class SqsHttpApi(object):
extract_result_method=lambda x: wrap_in_list(x['ListQueuesResponse']['ListQueuesResult']['QueueUrl']), default=(),
)
- def private_count_queues(self):
- return self.execute_request(
- action='CountQueues',
- private=True,
- extract_result_method=lambda x: x['CountQueuesResponse']['CountQueuesResult']['Count'],
- )
-
+ def private_count_queues(self):
+ return self.execute_request(
+ action='CountQueues',
+ private=True,
+ extract_result_method=lambda x: x['CountQueuesResponse']['CountQueuesResult']['Count'],
+ )
+
def create_queue(self, queue_name, is_fifo=False, attributes=None):
# if is_fifo and not queue_name.endswith('.fifo'):
# return None
@@ -265,13 +265,13 @@ class SqsHttpApi(object):
QueueName=queue_name
)
- def list_dead_letter_source_queues(self, queue_url):
- return self.execute_request(
- action='ListDeadLetterSourceQueues',
- extract_result_method=lambda x: wrap_in_list(x['ListDeadLetterSourceQueuesResponse']['ListDeadLetterSourceQueuesResult']['QueueUrl']),
- QueueUrl=queue_url
- )
-
+ def list_dead_letter_source_queues(self, queue_url):
+ return self.execute_request(
+ action='ListDeadLetterSourceQueues',
+ extract_result_method=lambda x: wrap_in_list(x['ListDeadLetterSourceQueuesResponse']['ListDeadLetterSourceQueuesResult']['QueueUrl']),
+ QueueUrl=queue_url
+ )
+
def get_queue_attributes(self, queue_url, attributes=['All']):
params = {}
for i in six.moves.range(len(attributes)):
@@ -283,9 +283,9 @@ class SqsHttpApi(object):
**params
)
result = {}
- if attrib_list is None:
- return result
-
+ if attrib_list is None:
+ return result
+
for attr in wrap_in_list(attrib_list):
result[attr['Name']] = attr['Value']
return result
@@ -313,30 +313,30 @@ class SqsHttpApi(object):
return result_list
- def modify_permissions(self, action, subject, path, permissions):
- args = {
- 'Path': path,
- 'Subject': subject
- }
- for i, permission in enumerate(permissions):
- args['Permission.' + str(i)] = permission
-
- return self.execute_request(
- action=action + 'Permissions',
- extract_result_method=lambda x: x['ModifyPermissionsResponse']['ResponseMetadata']['RequestId'],
- **args
- )
-
- def list_permissions(self, path):
- args = {
- 'Path': path,
- }
- return self.execute_request(
- action='ListPermissions',
- extract_result_method=lambda x: x['ListPermissionsResponse']['YaListPermissionsResult'],
- **args
- )
-
+ def modify_permissions(self, action, subject, path, permissions):
+ args = {
+ 'Path': path,
+ 'Subject': subject
+ }
+ for i, permission in enumerate(permissions):
+ args['Permission.' + str(i)] = permission
+
+ return self.execute_request(
+ action=action + 'Permissions',
+ extract_result_method=lambda x: x['ModifyPermissionsResponse']['ResponseMetadata']['RequestId'],
+ **args
+ )
+
+ def list_permissions(self, path):
+ args = {
+ 'Path': path,
+ }
+ return self.execute_request(
+ action='ListPermissions',
+ extract_result_method=lambda x: x['ListPermissionsResponse']['YaListPermissionsResult'],
+ **args
+ )
+
def set_queue_attributes(self, queue_url, attributes):
params = {}
i = 1
diff --git a/ydb/tests/functional/sqs/sqs_test_base.py b/ydb/tests/functional/sqs/sqs_test_base.py
index 83c07bf585..05d5303f64 100644
--- a/ydb/tests/functional/sqs/sqs_test_base.py
+++ b/ydb/tests/functional/sqs/sqs_test_base.py
@@ -11,7 +11,7 @@ import socket
from hamcrest import assert_that, equal_to, not_none, none, greater_than, less_than_or_equal_to, any_of, not_
import ydb.tests.library.common.yatest_common as yatest_common
-
+
from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
from ydb.tests.library.harness.util import LogLevels
@@ -35,12 +35,12 @@ IS_FIFO_PARAMS = {
'ids': ['fifo', 'std'],
}
-POLLING_PARAMS = {
- 'argnames': 'polling_wait_timeout',
- 'argvalues': [0, 1],
- 'ids': ['short_polling', 'long_polling'],
-}
-
+POLLING_PARAMS = {
+ 'argnames': 'polling_wait_timeout',
+ 'argvalues': [0, 1],
+ 'ids': ['short_polling', 'long_polling'],
+}
+
STOP_NODE_PARAMS = {
'argnames': 'stop_node',
'argvalues': [True, False],
@@ -182,15 +182,15 @@ class KikimrSqsTestBase(object):
max_queue_name_length = 80 - len('.fifo')
if len(self.queue_name) > max_queue_name_length:
self.queue_name = self.queue_name[:max_queue_name_length]
- self._msg_body_template = self._username + '-{}'
- self._setup_user(self._username)
+ self._msg_body_template = self._username + '-{}'
+ self._setup_user(self._username)
self._sqs_apis = []
for node in self.cluster_nodes:
self._sqs_apis.append(
SqsHttpApi(
'localhost',
node.sqs_port,
- self._username,
+ self._username,
raise_on_error=True,
timeout=None
)
@@ -312,7 +312,7 @@ class KikimrSqsTestBase(object):
'user',
'-u', 'metauser',
'-n', _username,
- ] + self._sqs_server_opts
+ ] + self._sqs_server_opts
while retries_count:
logging.debug("Running {}".format(' '.join(cmd)))
try:
@@ -325,17 +325,17 @@ class KikimrSqsTestBase(object):
return
raise RuntimeError("Failed to create SQS user")
- def _create_api_for_user(self, user_name, raise_on_error=True, security_token=None, force_private=False, iam_token=None, folder_id=None):
+ def _create_api_for_user(self, user_name, raise_on_error=True, security_token=None, force_private=False, iam_token=None, folder_id=None):
api = SqsHttpApi(self.cluster.nodes[1].host,
self.cluster_nodes[0].sqs_port,
user_name,
- raise_on_error=raise_on_error,
- timeout=None,
- security_token=security_token,
- force_private=force_private,
- iam_token=iam_token,
- folder_id=folder_id
- )
+ raise_on_error=raise_on_error,
+ timeout=None,
+ security_token=security_token,
+ force_private=force_private,
+ iam_token=iam_token,
+ folder_id=folder_id
+ )
return api
def _make_kikimr_driver(self, node_index=0):
@@ -386,8 +386,8 @@ class KikimrSqsTestBase(object):
if attributes is None:
attributes = dict()
logging.debug('Create queue. Attributes: {}. Use http: {}. Is fifo: {}'.format(attributes, use_http, is_fifo))
- assert (len(attributes.keys()) == 0 or use_http), 'Attributes are supported only for http queue creation'
- assert (shards is None or not use_http), 'Custom shards number is only supported in non-http mode'
+ assert (len(attributes.keys()) == 0 or use_http), 'Attributes are supported only for http queue creation'
+ assert (shards is None or not use_http), 'Custom shards number is only supported in non-http mode'
while retries:
retries -= 1
try:
@@ -397,11 +397,11 @@ class KikimrSqsTestBase(object):
cmd = [
get_sqs_client_path(),
'create', # create queue command
- '-u', self._username,
- '--shards', str(shards) if shards is not None else '2',
+ '-u', self._username,
+ '--shards', str(shards) if shards is not None else '2',
'--partitions', '1',
'--queue-name', queue_name,
- ] + self._sqs_server_opts
+ ] + self._sqs_server_opts
execute = yatest_common.execute(cmd)
self.queue_url = execute.std_out
self.queue_url = self.queue_url.strip()
@@ -626,29 +626,29 @@ class KikimrSqsTestBase(object):
logging.debug('Connection error: trying to enable tablets on node {} in 1 second'.format(node_index))
time.sleep(1) # wait node to start
- def _smart_make_table_path(self, user_name, queue_name, queue_version, shard, table_name):
+ def _smart_make_table_path(self, user_name, queue_name, queue_version, shard, table_name):
table_path = '{}/{}'.format(self.sqs_root, user_name)
- if queue_name is not None:
- table_path += '/{}'.format(queue_name)
- if queue_version is not None and queue_version != 0:
- table_path += '/v{}'.format(queue_version)
- if shard is not None:
- table_path += '/{}'.format(shard)
-
- return table_path + '/{}'.format(table_name)
-
- def _get_queue_version_number(self, user_name, queue_name):
+ if queue_name is not None:
+ table_path += '/{}'.format(queue_name)
+ if queue_version is not None and queue_version != 0:
+ table_path += '/v{}'.format(queue_version)
+ if shard is not None:
+ table_path += '/{}'.format(shard)
+
+ return table_path + '/{}'.format(table_name)
+
+ def _get_queue_version_number(self, user_name, queue_name):
table_path = '{}/.Queues'.format(self.sqs_root)
data_result_sets = self._execute_yql_query('SELECT Version FROM `{}` WHERE Account=\'{}\' AND QueueName=\'{}\''.format(table_path, user_name, queue_name))
- assert_that(len(data_result_sets), equal_to(1))
- assert_that(len(data_result_sets[0].rows), equal_to(1))
-
- version = data_result_sets[0].rows[0]['Version']
- if version is None:
- return 0
- else:
- return int(version)
-
+ assert_that(len(data_result_sets), equal_to(1))
+ assert_that(len(data_result_sets[0].rows), equal_to(1))
+
+ version = data_result_sets[0].rows[0]['Version']
+ if version is None:
+ return 0
+ else:
+ return int(version)
+
def _get_queue_master_tablet_id(self, user_name_param=None, queue_name_param=None):
user_name = user_name_param if user_name_param else self._username
queue_name = queue_name_param if queue_name_param else self.queue_name
@@ -696,12 +696,12 @@ class KikimrSqsTestBase(object):
queue_name = self.queue_name
is_fifo = queue_name.endswith('.fifo')
- queue_version = self._get_queue_version_number(self._username, queue_name)
-
+ queue_version = self._get_queue_version_number(self._username, queue_name)
+
if is_fifo:
- self._check_fifo_queue_is_empty(queue_name, queue_version)
+ self._check_fifo_queue_is_empty(queue_name, queue_version)
else:
- self._check_std_queue_is_empty(queue_name, queue_version)
+ self._check_std_queue_is_empty(queue_name, queue_version)
def _get_table_lines_count(self, table_path):
data_result_sets = self._execute_yql_query('SELECT COUNT(*) AS count FROM `{}`;'.format(table_path))
@@ -710,15 +710,15 @@ class KikimrSqsTestBase(object):
logging.debug('Received count result for table {}: {}'.format(table_path, data_result_sets[0].rows[0]))
return data_result_sets[0].rows[0]['count']
- def _check_std_queue_is_empty(self, queue_name, queue_version):
+ def _check_std_queue_is_empty(self, queue_name, queue_version):
shards = self._get_queue_shards_count(self._username, queue_name, queue_version)
assert_that(shards, not_(equal_to(0)))
for shard in range(shards):
- self._check_std_queue_shard_is_empty(queue_name, queue_version, shard)
+ self._check_std_queue_shard_is_empty(queue_name, queue_version, shard)
- def _check_std_queue_shard_is_empty(self, queue_name, queue_version, shard):
+ def _check_std_queue_shard_is_empty(self, queue_name, queue_version, shard):
def get_table_path(table_name):
- return self._smart_make_table_path(self._username, queue_name, queue_version, shard, table_name)
+ return self._smart_make_table_path(self._username, queue_name, queue_version, shard, table_name)
def get_lines_count(table_name):
return self._get_table_lines_count(get_table_path(table_name))
@@ -728,9 +728,9 @@ class KikimrSqsTestBase(object):
assert_that(get_lines_count('Messages'), equal_to(0))
assert_that(get_lines_count('SentTimestampIdx'), equal_to(0))
- def _check_fifo_queue_is_empty(self, queue_name, queue_version):
+ def _check_fifo_queue_is_empty(self, queue_name, queue_version):
def get_table_path(table_name):
- return self._smart_make_table_path(self._username, queue_name, queue_version, None, table_name)
+ return self._smart_make_table_path(self._username, queue_name, queue_version, None, table_name)
def get_lines_count(table_name):
return self._get_table_lines_count(get_table_path(table_name))
diff --git a/ydb/tests/functional/sqs/test_account_actions.py b/ydb/tests/functional/sqs/test_account_actions.py
index 1b7e733534..60ab1e4d10 100644
--- a/ydb/tests/functional/sqs/test_account_actions.py
+++ b/ydb/tests/functional/sqs/test_account_actions.py
@@ -4,29 +4,29 @@ from hamcrest import assert_that, not_none, has_item, is_not
from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation
-
+
class AccountActionsTest(KikimrSqsTestBase):
- def test_manage_account(self):
- user_name = 'pupkin'
- create_user_result = self._sqs_api.create_user(user_name)
- assert_that(
- create_user_result,
- not_none()
- )
- user_list = self._sqs_api.list_users()
- assert_that(
- user_list, has_item(user_name)
- )
-
- delete_user_result = self._sqs_api.delete_user(user_name)
- assert_that(
- delete_user_result,
- not_none()
- )
- user_list = self._sqs_api.list_users()
- assert_that(
- user_list, is_not(has_item(user_name))
- )
+ def test_manage_account(self):
+ user_name = 'pupkin'
+ create_user_result = self._sqs_api.create_user(user_name)
+ assert_that(
+ create_user_result,
+ not_none()
+ )
+ user_list = self._sqs_api.list_users()
+ assert_that(
+ user_list, has_item(user_name)
+ )
+
+ delete_user_result = self._sqs_api.delete_user(user_name)
+ assert_that(
+ delete_user_result,
+ not_none()
+ )
+ user_list = self._sqs_api.list_users()
+ assert_that(
+ user_list, is_not(has_item(user_name))
+ )
class TestAccountActionsWithTenant(get_test_with_sqs_tenant_installation(AccountActionsTest)):
diff --git a/ydb/tests/functional/sqs/test_acl.py b/ydb/tests/functional/sqs/test_acl.py
index 658fcebcb6..e5221f0f43 100644
--- a/ydb/tests/functional/sqs/test_acl.py
+++ b/ydb/tests/functional/sqs/test_acl.py
@@ -7,167 +7,167 @@ import pytest
from hamcrest import assert_that, equal_to, none, is_not, is_, raises
import ydb.tests.library.common.yatest_common as yatest_common
-
+
from sqs_test_base import KikimrSqsTestBase, get_sqs_client_path, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, to_bytes
class SqsACLTest(KikimrSqsTestBase):
- def _modify_permissions(self, resource, action, permissions_string, clear_acl_flag=False):
- cmd = [
- get_sqs_client_path(),
- 'permissions',
- '--resource', resource,
- '--{}'.format(action),
- '{}'.format(permissions_string)
- ]
- if clear_acl_flag:
- cmd.append('--clear-acl')
-
- cmd += self._sqs_server_opts
-
- retries_count = 1
- while retries_count:
+ def _modify_permissions(self, resource, action, permissions_string, clear_acl_flag=False):
+ cmd = [
+ get_sqs_client_path(),
+ 'permissions',
+ '--resource', resource,
+ '--{}'.format(action),
+ '{}'.format(permissions_string)
+ ]
+ if clear_acl_flag:
+ cmd.append('--clear-acl')
+
+ cmd += self._sqs_server_opts
+
+ retries_count = 1
+ while retries_count:
logging.debug("Running {}".format(' '.join(cmd)))
- try:
- yatest_common.execute(cmd)
- except yatest_common.ExecutionError as ex:
- logging.debug("Modify permissions failed: {}. Retrying".format(ex))
- retries_count -= 1
- time.sleep(3)
- else:
- return
- raise RuntimeError("Failed to modify permissions")
-
- def _list_permissions(self, path, retries_count=1):
- cmd = [
- get_sqs_client_path(),
- 'list-permissions',
- '--path', path,
- ] + self._sqs_server_opts
-
- while retries_count:
+ try:
+ yatest_common.execute(cmd)
+ except yatest_common.ExecutionError as ex:
+ logging.debug("Modify permissions failed: {}. Retrying".format(ex))
+ retries_count -= 1
+ time.sleep(3)
+ else:
+ return
+ raise RuntimeError("Failed to modify permissions")
+
+ def _list_permissions(self, path, retries_count=1):
+ cmd = [
+ get_sqs_client_path(),
+ 'list-permissions',
+ '--path', path,
+ ] + self._sqs_server_opts
+
+ while retries_count:
logging.debug("Running {}".format(' '.join(cmd)))
- try:
- execute = yatest_common.execute(cmd)
- except yatest_common.ExecutionError as ex:
- logging.debug("List permissions failed: {}. Retrying".format(ex))
- retries_count -= 1
- time.sleep(3)
- else:
- return execute.std_out
- raise RuntimeError("Failed to list permissions")
-
- def __generate_expected_simplified_permissions_response(self, sid=None, permission_names=[]):
- resp = 'AccountPermissions {'
- if not permission_names:
- resp += '}'
- else:
- resp += 'Permissions {{ Subject: \"{}\"'.format(sid)
- for permission_name in permission_names:
- resp += 'PermissionNames: \"{}\"'.format(permission_name)
- resp += '}} EffectivePermissions {{ Subject: \"{}\"'.format(sid)
- for permission_name in permission_names:
- resp += 'PermissionNames: \"{}\"'.format(permission_name)
- resp += '}}'
-
- return resp
-
- def __clean_and_compare(self, s1, s2):
+ try:
+ execute = yatest_common.execute(cmd)
+ except yatest_common.ExecutionError as ex:
+ logging.debug("List permissions failed: {}. Retrying".format(ex))
+ retries_count -= 1
+ time.sleep(3)
+ else:
+ return execute.std_out
+ raise RuntimeError("Failed to list permissions")
+
+ def __generate_expected_simplified_permissions_response(self, sid=None, permission_names=[]):
+ resp = 'AccountPermissions {'
+ if not permission_names:
+ resp += '}'
+ else:
+ resp += 'Permissions {{ Subject: \"{}\"'.format(sid)
+ for permission_name in permission_names:
+ resp += 'PermissionNames: \"{}\"'.format(permission_name)
+ resp += '}} EffectivePermissions {{ Subject: \"{}\"'.format(sid)
+ for permission_name in permission_names:
+ resp += 'PermissionNames: \"{}\"'.format(permission_name)
+ resp += '}}'
+
+ return resp
+
+ def __clean_and_compare(self, s1, s2):
s1 = to_bytes(s1)[to_bytes(s1).find(to_bytes('AccountPermissions')):].replace(to_bytes('\n'), to_bytes('')).replace(to_bytes(' '), to_bytes(''))
s2 = to_bytes(s2)[to_bytes(s2).find(to_bytes('AccountPermissions')):].replace(to_bytes('\n'), to_bytes('')).replace(to_bytes(' '), to_bytes(''))
-
- assert_that(s1, equal_to(s2))
-
- def test_modify_permissions(self):
- queue_url = self._create_queue_and_assert(self.queue_name, False, True)
- self._send_message_and_assert(queue_url, 'data')
-
- alkonavt_sid = 'alkonavt@builtin'
- berkanavt_sid = 'berkanavt@builtin'
-
- create_queue_permission = 'CreateQueue'
- send_message_permission = 'SendMessage'
-
- # no permissions expected
- description = self._list_permissions(self._username)
- expected_description = self.__generate_expected_simplified_permissions_response()
- self.__clean_and_compare(description, expected_description)
-
- # two permissions expected
- self._modify_permissions(
- self._username,
- 'grant',
- alkonavt_sid + ':' + ','.join([create_queue_permission, send_message_permission])
- )
- description = self._list_permissions(self._username)
- expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [create_queue_permission, send_message_permission])
- self.__clean_and_compare(description, expected_description)
-
- # single permission expected
- self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission)
- description = self._list_permissions(self._username)
- expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [send_message_permission])
- self.__clean_and_compare(description, expected_description)
-
- receive_message_permission = 'ReceiveMessage'
-
- # other single permission expected
- self._modify_permissions(self._username, 'set', alkonavt_sid + ':' + receive_message_permission)
- description = self._list_permissions(self._username)
- expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [receive_message_permission])
- self.__clean_and_compare(description, expected_description)
-
- # clear all permissions
- self._modify_permissions(self._username, 'set', berkanavt_sid + ':' + receive_message_permission)
- self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission, True)
- description = self._list_permissions(self._username)
- expected_description = self.__generate_expected_simplified_permissions_response()
- self.__clean_and_compare(description, expected_description)
-
- def test_apply_permissions(self):
- queue_url = self._create_queue_and_assert(self.queue_name, False, True)
- self._send_message_and_assert(queue_url, 'data')
-
- berkanavt_sid = 'berkanavt@builtin'
-
- self._sqs_api = self._create_api_for_user(self._username, raise_on_error=False, security_token=berkanavt_sid)
-
- def __send_message_with_retries(_queue_url, data, result_predicate):
- retries = 3
-
- while retries > 0:
- retries = retries - 1
- try:
- result = self._sqs_api.send_message(_queue_url, data)
- assert_that(result, result_predicate)
- return result
- except:
- if retries == 0:
- raise
- time.sleep(0.1)
-
- __send_message_with_retries(queue_url, 'megadata', is_(none())) # no access
-
- self._modify_permissions(
- self._username,
- 'grant',
- berkanavt_sid + ':' + ','.join(['ModifyPermissions'])
- )
-
- result = self._sqs_api.modify_permissions('Grant', berkanavt_sid, self._username, ['SendMessage', 'DescribePath'])
- assert_that(result, is_not(none()))
-
- __send_message_with_retries(queue_url, 'utradata', is_not(none())) # has access
-
- result = self._sqs_api.modify_permissions('Revoke', berkanavt_sid, self._username, ['SendMessage', 'AlterQueue'])
- assert_that(result, is_not(none()))
-
- __send_message_with_retries(queue_url, 'superdata', is_(none())) # no access again. that's a pity
-
- result = self._sqs_api.list_permissions(self._username)
- assert('Account' in str(result))
- assert(berkanavt_sid in str(result))
- assert('Permissions' in str(result))
+
+ assert_that(s1, equal_to(s2))
+
+ def test_modify_permissions(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, False, True)
+ self._send_message_and_assert(queue_url, 'data')
+
+ alkonavt_sid = 'alkonavt@builtin'
+ berkanavt_sid = 'berkanavt@builtin'
+
+ create_queue_permission = 'CreateQueue'
+ send_message_permission = 'SendMessage'
+
+ # no permissions expected
+ description = self._list_permissions(self._username)
+ expected_description = self.__generate_expected_simplified_permissions_response()
+ self.__clean_and_compare(description, expected_description)
+
+ # two permissions expected
+ self._modify_permissions(
+ self._username,
+ 'grant',
+ alkonavt_sid + ':' + ','.join([create_queue_permission, send_message_permission])
+ )
+ description = self._list_permissions(self._username)
+ expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [create_queue_permission, send_message_permission])
+ self.__clean_and_compare(description, expected_description)
+
+ # single permission expected
+ self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission)
+ description = self._list_permissions(self._username)
+ expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [send_message_permission])
+ self.__clean_and_compare(description, expected_description)
+
+ receive_message_permission = 'ReceiveMessage'
+
+ # other single permission expected
+ self._modify_permissions(self._username, 'set', alkonavt_sid + ':' + receive_message_permission)
+ description = self._list_permissions(self._username)
+ expected_description = self.__generate_expected_simplified_permissions_response(alkonavt_sid, [receive_message_permission])
+ self.__clean_and_compare(description, expected_description)
+
+ # clear all permissions
+ self._modify_permissions(self._username, 'set', berkanavt_sid + ':' + receive_message_permission)
+ self._modify_permissions(self._username, 'revoke', alkonavt_sid + ':' + create_queue_permission, True)
+ description = self._list_permissions(self._username)
+ expected_description = self.__generate_expected_simplified_permissions_response()
+ self.__clean_and_compare(description, expected_description)
+
+ def test_apply_permissions(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, False, True)
+ self._send_message_and_assert(queue_url, 'data')
+
+ berkanavt_sid = 'berkanavt@builtin'
+
+ self._sqs_api = self._create_api_for_user(self._username, raise_on_error=False, security_token=berkanavt_sid)
+
+ def __send_message_with_retries(_queue_url, data, result_predicate):
+ retries = 3
+
+ while retries > 0:
+ retries = retries - 1
+ try:
+ result = self._sqs_api.send_message(_queue_url, data)
+ assert_that(result, result_predicate)
+ return result
+ except:
+ if retries == 0:
+ raise
+ time.sleep(0.1)
+
+ __send_message_with_retries(queue_url, 'megadata', is_(none())) # no access
+
+ self._modify_permissions(
+ self._username,
+ 'grant',
+ berkanavt_sid + ':' + ','.join(['ModifyPermissions'])
+ )
+
+ result = self._sqs_api.modify_permissions('Grant', berkanavt_sid, self._username, ['SendMessage', 'DescribePath'])
+ assert_that(result, is_not(none()))
+
+ __send_message_with_retries(queue_url, 'utradata', is_not(none())) # has access
+
+ result = self._sqs_api.modify_permissions('Revoke', berkanavt_sid, self._username, ['SendMessage', 'AlterQueue'])
+ assert_that(result, is_not(none()))
+
+ __send_message_with_retries(queue_url, 'superdata', is_(none())) # no access again. that's a pity
+
+ result = self._sqs_api.list_permissions(self._username)
+ assert('Account' in str(result))
+ assert(berkanavt_sid in str(result))
+ assert('Permissions' in str(result))
class SqsWithForceAuthorizationTest(KikimrSqsTestBase):
diff --git a/ydb/tests/functional/sqs/test_fifo_messaging.py b/ydb/tests/functional/sqs/test_fifo_messaging.py
index 14d96ed93e..6410b21966 100644
--- a/ydb/tests/functional/sqs/test_fifo_messaging.py
+++ b/ydb/tests/functional/sqs/test_fifo_messaging.py
@@ -11,35 +11,35 @@ from sqs_matchers import ReadResponseMatcher, extract_message_ids
from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, VISIBILITY_CHANGE_METHOD_PARAMS
-class SqsFifoMicroBatchTest(KikimrSqsTestBase):
- @classmethod
- def _setup_config_generator(cls):
- config_generator = super(SqsFifoMicroBatchTest, cls)._setup_config_generator()
+class SqsFifoMicroBatchTest(KikimrSqsTestBase):
+ @classmethod
+ def _setup_config_generator(cls):
+ config_generator = super(SqsFifoMicroBatchTest, cls)._setup_config_generator()
config_generator.yaml_config['sqs_config']['group_selection_batch_size'] = 2
- return config_generator
-
- def test_micro_batch_read(self):
- self.queue_name = self.queue_name + '.fifo'
- created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
- seq_no = 0
- max_number_of_messages = 3
- for i in range(max_number_of_messages):
- self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=str(seq_no), group_id=str(seq_no))
- seq_no += 1
-
- for i in range(2):
- msgs = self._sqs_api.receive_message(created_queue_url, max_number_of_messages=max_number_of_messages, visibility_timeout=0, receive_request_attempt_id='test')
- assert_that(len(set([msgs[i]['MessageId'] for i in range(len(msgs))])), equal_to(len(msgs)))
-
-
-class TestSqsFifoMicroBatchesWithTenant(get_test_with_sqs_tenant_installation(SqsFifoMicroBatchTest)):
- pass
-
-
-class TestSqsFifoMicroBatchesWithPath(get_test_with_sqs_installation_by_path(SqsFifoMicroBatchTest)):
- pass
-
-
+ return config_generator
+
+ def test_micro_batch_read(self):
+ self.queue_name = self.queue_name + '.fifo'
+ created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
+ seq_no = 0
+ max_number_of_messages = 3
+ for i in range(max_number_of_messages):
+ self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=str(seq_no), group_id=str(seq_no))
+ seq_no += 1
+
+ for i in range(2):
+ msgs = self._sqs_api.receive_message(created_queue_url, max_number_of_messages=max_number_of_messages, visibility_timeout=0, receive_request_attempt_id='test')
+ assert_that(len(set([msgs[i]['MessageId'] for i in range(len(msgs))])), equal_to(len(msgs)))
+
+
+class TestSqsFifoMicroBatchesWithTenant(get_test_with_sqs_tenant_installation(SqsFifoMicroBatchTest)):
+ pass
+
+
+class TestSqsFifoMicroBatchesWithPath(get_test_with_sqs_installation_by_path(SqsFifoMicroBatchTest)):
+ pass
+
+
class SqsFifoMessagingTest(KikimrSqsTestBase):
def setup_method(self, method=None):
super(SqsFifoMessagingTest, self).setup_method(method)
diff --git a/ydb/tests/functional/sqs/test_garbage_collection.py b/ydb/tests/functional/sqs/test_garbage_collection.py
index 919ad29705..992c8a6513 100644
--- a/ydb/tests/functional/sqs/test_garbage_collection.py
+++ b/ydb/tests/functional/sqs/test_garbage_collection.py
@@ -9,7 +9,7 @@ import pytest
from hamcrest import assert_that, equal_to, less_than_or_equal_to
from sqs_requests_client import SqsHttpApi
-
+
from sqs_matchers import ReadResponseMatcher
from sqs_test_base import to_bytes
diff --git a/ydb/tests/functional/sqs/test_generic_messaging.py b/ydb/tests/functional/sqs/test_generic_messaging.py
index 1546a2b0e7..3f6b2371ba 100644
--- a/ydb/tests/functional/sqs/test_generic_messaging.py
+++ b/ydb/tests/functional/sqs/test_generic_messaging.py
@@ -9,7 +9,7 @@ import pytest
from hamcrest import assert_that, equal_to, not_none, greater_than, has_item, has_items, raises, is_not, not_, empty, instance_of
from sqs_requests_client import SqsMessageAttribute, SqsSendMessageParams, SqsChangeMessageVisibilityParams
-
+
from sqs_matchers import ReadResponseMatcher, extract_message_ids
from sqs_test_base import to_bytes
from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS
@@ -644,10 +644,10 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
assert_that(
self._sqs_api.delete_message(self.queue_url, handle), not_none()
)
- # check double deletion
- assert_that(
- self._sqs_api.delete_message(self.queue_url, handle), not_none()
- )
+ # check double deletion
+ assert_that(
+ self._sqs_api.delete_message(self.queue_url, handle), not_none()
+ )
self._read_messages_and_assert(
self.queue_url, messages_count=10, visibility_timeout=1000,
matcher=ReadResponseMatcher().with_message_ids(self.message_ids)
@@ -849,13 +849,13 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
if is_fifo:
self.queue_name = self.queue_name + '.fifo'
queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
-
- # assert empty response when no attribute names are provided
- attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=[])
- assert_that(attributes, equal_to({}))
-
- # check common case
- attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=['All'])
+
+ # assert empty response when no attribute names are provided
+ attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=[])
+ assert_that(attributes, equal_to({}))
+
+ # check common case
+ attributes = self._sqs_api.get_queue_attributes(queue_url, attributes=['All'])
if is_fifo:
assert_that(attributes, has_item('FifoQueue'))
assert_that(attributes, has_item('ContentBasedDeduplication'))
@@ -876,7 +876,7 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
))
assert_that(attributes['ReceiveMessageWaitTimeSeconds'], equal_to('0'))
if is_fifo:
- assert_that(attributes['ContentBasedDeduplication'], equal_to('false'))
+ assert_that(attributes['ContentBasedDeduplication'], equal_to('false'))
self._sqs_api.set_queue_attributes(queue_url, {'ReceiveMessageWaitTimeSeconds': '10', 'MaximumMessageSize': '111111'})
attributes = self._sqs_api.get_queue_attributes(queue_url)
@@ -886,7 +886,7 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
if is_fifo:
self._sqs_api.set_queue_attributes(queue_url, {'ContentBasedDeduplication': 'true'})
attributes = self._sqs_api.get_queue_attributes(queue_url)
- assert_that(attributes['ContentBasedDeduplication'], equal_to('true'))
+ assert_that(attributes['ContentBasedDeduplication'], equal_to('true'))
def test_set_very_big_visibility_timeout(self):
queue_url = self._create_queue_and_assert(self.queue_name)
@@ -896,7 +896,7 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
assert_that(
call_with_very_big_visibility_timeout,
- raises(RuntimeError, pattern='InvalidAttributeValue')
+ raises(RuntimeError, pattern='InvalidAttributeValue')
)
def test_wrong_attribute_name(self):
diff --git a/ydb/tests/functional/sqs/test_multinode_cluster.py b/ydb/tests/functional/sqs/test_multinode_cluster.py
index 4a27391ef2..047ae56198 100644
--- a/ydb/tests/functional/sqs/test_multinode_cluster.py
+++ b/ydb/tests/functional/sqs/test_multinode_cluster.py
@@ -8,7 +8,7 @@ import pytest
from hamcrest import assert_that, equal_to, not_none, raises, not_
from ydb.tests.library.common.types import Erasure
-
+
from sqs_matchers import ReadResponseMatcher
from sqs_test_base import KikimrSqsTestBase, STOP_NODE_PARAMS, IS_FIFO_PARAMS
diff --git a/ydb/tests/functional/sqs/test_polling.py b/ydb/tests/functional/sqs/test_polling.py
index 7f972cb169..dba0556e7d 100644
--- a/ydb/tests/functional/sqs/test_polling.py
+++ b/ydb/tests/functional/sqs/test_polling.py
@@ -4,36 +4,36 @@ import pytest
from hamcrest import assert_that, equal_to
from sqs_matchers import ReadResponseMatcher
-
+
from sqs_test_base import KikimrSqsTestBase, POLLING_PARAMS, IS_FIFO_PARAMS
-class TestSqsPolling(KikimrSqsTestBase):
- @classmethod
- def _setup_config_generator(cls):
- config_generator = super(TestSqsPolling, cls)._setup_config_generator()
+class TestSqsPolling(KikimrSqsTestBase):
+ @classmethod
+ def _setup_config_generator(cls):
+ config_generator = super(TestSqsPolling, cls)._setup_config_generator()
config_generator.yaml_config['sqs_config']['check_all_shards_in_receive_message'] = False
- return config_generator
-
- @pytest.mark.parametrize(**IS_FIFO_PARAMS)
- @pytest.mark.parametrize(**POLLING_PARAMS)
- def test_receive_message_with_polling(self, is_fifo, polling_wait_timeout):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
- created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1)
- empty_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1)
-
- self.seq_no += 1
- self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None)
-
- read_result = self._read_messages_and_assert(
- created_queue_url, messages_count=1, visibility_timeout=1000, wait_timeout=polling_wait_timeout,
+ return config_generator
+
+ @pytest.mark.parametrize(**IS_FIFO_PARAMS)
+ @pytest.mark.parametrize(**POLLING_PARAMS)
+ def test_receive_message_with_polling(self, is_fifo, polling_wait_timeout):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+ created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1)
+ empty_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=False, shards=None if is_fifo else 1)
+
+ self.seq_no += 1
+ self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None)
+
+ read_result = self._read_messages_and_assert(
+ created_queue_url, messages_count=1, visibility_timeout=1000, wait_timeout=polling_wait_timeout,
matcher=ReadResponseMatcher().with_n_messages(1)
- )
- assert_that(read_result[0]['Body'], equal_to('test_send_message'))
-
- # check that there's no infinite loop for empty queue
- read_result = self._read_messages_and_assert(
- empty_queue_url, messages_count=3, visibility_timeout=1000, wait_timeout=polling_wait_timeout
- )
- assert_that(len(read_result), equal_to(0))
+ )
+ assert_that(read_result[0]['Body'], equal_to('test_send_message'))
+
+ # check that there's no infinite loop for empty queue
+ read_result = self._read_messages_and_assert(
+ empty_queue_url, messages_count=3, visibility_timeout=1000, wait_timeout=polling_wait_timeout
+ )
+ assert_that(len(read_result), equal_to(0))
diff --git a/ydb/tests/functional/sqs/test_queue_attributes_validation.py b/ydb/tests/functional/sqs/test_queue_attributes_validation.py
index e7e2ea8723..27ec79c9ee 100644
--- a/ydb/tests/functional/sqs/test_queue_attributes_validation.py
+++ b/ydb/tests/functional/sqs/test_queue_attributes_validation.py
@@ -4,183 +4,183 @@ import pytest
from hamcrest import assert_that, equal_to
from sqs_test_base import KikimrSqsTestBase, IS_FIFO_PARAMS
-
-class TestQueueAttributesInCompatibilityMode(KikimrSqsTestBase):
- @classmethod
- def _setup_config_generator(cls):
- config_generator = super(TestQueueAttributesInCompatibilityMode, cls)._setup_config_generator()
+
+class TestQueueAttributesInCompatibilityMode(KikimrSqsTestBase):
+ @classmethod
+ def _setup_config_generator(cls):
+ config_generator = super(TestQueueAttributesInCompatibilityMode, cls)._setup_config_generator()
config_generator.yaml_config['sqs_config']['enable_queue_attributes_validation'] = False
- return config_generator
-
- @pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_set_queue_attributes_no_validation(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True, attributes={'MaximumMessageSize': '1000000'})
- assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to(str(256 * 1024)))
-
- self._sqs_api.set_queue_attributes(queue_url, {'MaximumMessageSize': '1'})
- assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024'))
-
- # create queue again, this should yield no error
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True)
- # previously set attributes should be right there
- assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024'))
- try:
+ return config_generator
+
+ @pytest.mark.parametrize(**IS_FIFO_PARAMS)
+ def test_set_queue_attributes_no_validation(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True, attributes={'MaximumMessageSize': '1000000'})
+ assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to(str(256 * 1024)))
+
+ self._sqs_api.set_queue_attributes(queue_url, {'MaximumMessageSize': '1'})
+ assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024'))
+
+ # create queue again, this should yield no error
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True)
+ # previously set attributes should be right there
+ assert_that(self._sqs_api.get_queue_attributes(queue_url)['MaximumMessageSize'], equal_to('1024'))
+ try:
queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True, attributes={'MaximumMessageSize': 'troll'}, retries=1)
- except:
- pass
- else:
- raise Exception('only parseable attributes are valid')
-
-
-class TestQueueAttributesValidation(KikimrSqsTestBase):
- @classmethod
- def _setup_config_generator(cls):
- config_generator = super(TestQueueAttributesValidation, cls)._setup_config_generator()
- return config_generator
-
- @pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_set_queue_attributes(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True)
-
- attributes = self._sqs_api.get_queue_attributes(queue_url)
- queue_attributes_list = ['VisibilityTimeout', 'MaximumMessageSize', 'DelaySeconds', 'MessageRetentionPeriod', 'ReceiveMessageWaitTimeSeconds']
- attributes_to_set_and_check = {}
- for attr in queue_attributes_list:
- val = int(attributes[attr])
- # change attributes slightly
- attributes_to_set_and_check[attr] = str(val - 1 if val > 0 else val + 1)
- assert(attributes_to_set_and_check[attr] != attributes[attr])
-
- content_based_dedup_key = 'ContentBasedDeduplication'
- if is_fifo:
- val = attributes[content_based_dedup_key]
- attributes_to_set_and_check[content_based_dedup_key] = 'true' if val == 'false' else 'false'
- assert(attributes_to_set_and_check[content_based_dedup_key] != attributes[content_based_dedup_key])
-
- self._sqs_api.set_queue_attributes(queue_url, attributes_to_set_and_check)
- # match set attributes
- attributes = self._sqs_api.get_queue_attributes(queue_url)
- for attr in attributes_to_set_and_check:
- assert_that(attributes_to_set_and_check[attr], equal_to(attributes[attr]))
-
- # okay, now we'll try to break it
- for attr in attributes_to_set_and_check:
- try:
- self._sqs_api.set_queue_attributes(queue_url, {attr: '2.5'})
+ except:
+ pass
+ else:
+ raise Exception('only parseable attributes are valid')
+
+
+class TestQueueAttributesValidation(KikimrSqsTestBase):
+ @classmethod
+ def _setup_config_generator(cls):
+ config_generator = super(TestQueueAttributesValidation, cls)._setup_config_generator()
+ return config_generator
+
+ @pytest.mark.parametrize(**IS_FIFO_PARAMS)
+ def test_set_queue_attributes(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True)
+
+ attributes = self._sqs_api.get_queue_attributes(queue_url)
+ queue_attributes_list = ['VisibilityTimeout', 'MaximumMessageSize', 'DelaySeconds', 'MessageRetentionPeriod', 'ReceiveMessageWaitTimeSeconds']
+ attributes_to_set_and_check = {}
+ for attr in queue_attributes_list:
+ val = int(attributes[attr])
+ # change attributes slightly
+ attributes_to_set_and_check[attr] = str(val - 1 if val > 0 else val + 1)
+ assert(attributes_to_set_and_check[attr] != attributes[attr])
+
+ content_based_dedup_key = 'ContentBasedDeduplication'
+ if is_fifo:
+ val = attributes[content_based_dedup_key]
+ attributes_to_set_and_check[content_based_dedup_key] = 'true' if val == 'false' else 'false'
+ assert(attributes_to_set_and_check[content_based_dedup_key] != attributes[content_based_dedup_key])
+
+ self._sqs_api.set_queue_attributes(queue_url, attributes_to_set_and_check)
+ # match set attributes
+ attributes = self._sqs_api.get_queue_attributes(queue_url)
+ for attr in attributes_to_set_and_check:
+ assert_that(attributes_to_set_and_check[attr], equal_to(attributes[attr]))
+
+ # okay, now we'll try to break it
+ for attr in attributes_to_set_and_check:
+ try:
+ self._sqs_api.set_queue_attributes(queue_url, {attr: '2.5'})
except Exception as e:
- assert('InvalidAttributeValue' in str(e))
- assert(attr in str(e))
- else:
- assert_that(False) # expected InvalidAttributeValue exception
-
- try:
- self._sqs_api.set_queue_attributes(queue_url, {attr: '2147483647'})
+ assert('InvalidAttributeValue' in str(e))
+ assert(attr in str(e))
+ else:
+ assert_that(False) # expected InvalidAttributeValue exception
+
+ try:
+ self._sqs_api.set_queue_attributes(queue_url, {attr: '2147483647'})
except Exception as e:
- assert('InvalidAttributeValue' in str(e))
- assert(attr in str(e))
- else:
- assert_that(False) # expected InvalidAttributeValue exception
-
- try:
- self._sqs_api.set_queue_attributes(queue_url, {'trololo': 'ololo'})
+ assert('InvalidAttributeValue' in str(e))
+ assert(attr in str(e))
+ else:
+ assert_that(False) # expected InvalidAttributeValue exception
+
+ try:
+ self._sqs_api.set_queue_attributes(queue_url, {'trololo': 'ololo'})
except Exception as e:
- assert('InvalidAttributeName' in str(e))
- else:
- assert_that(False) # expected InvalidAttributeName exception
-
- # check setting FifoQueue attribute
- if is_fifo:
- self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'true'}) # ok, do nothing
- try:
- self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'omg'})
+ assert('InvalidAttributeName' in str(e))
+ else:
+ assert_that(False) # expected InvalidAttributeName exception
+
+ # check setting FifoQueue attribute
+ if is_fifo:
+ self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'true'}) # ok, do nothing
+ try:
+ self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'omg'})
except Exception as e:
- assert('InvalidAttributeValue' in str(e))
- assert('FifoQueue' in str(e))
- else:
- assert_that(False) # expected InvalidAttributeValue exception
-
- # special case: queue type mismatch
- try:
- self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'})
+ assert('InvalidAttributeValue' in str(e))
+ assert('FifoQueue' in str(e))
+ else:
+ assert_that(False) # expected InvalidAttributeValue exception
+
+ # special case: queue type mismatch
+ try:
+ self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'})
except Exception as e:
- assert('InvalidAttributeValue' in str(e))
- assert('Modifying queue type is not supported' in str(e))
- else:
- assert_that(False) # expected InvalidAttributeValue exception
- else:
- try:
- self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'})
+ assert('InvalidAttributeValue' in str(e))
+ assert('Modifying queue type is not supported' in str(e))
+ else:
+ assert_that(False) # expected InvalidAttributeValue exception
+ else:
+ try:
+ self._sqs_api.set_queue_attributes(queue_url, {'FifoQueue': 'false'})
except Exception as e:
- assert('InvalidAttributeName' in str(e))
- else:
- assert_that(False) # expected InvalidAttributeName exception
-
- @pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_create_queue_with_default_attributes(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
-
- # check default queue creation
- queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo)
-
- expected_attributes = {
- 'DelaySeconds': '0',
- 'MaximumMessageSize': '262144',
- 'MessageRetentionPeriod': '345600',
- 'ReceiveMessageWaitTimeSeconds': '0',
- 'VisibilityTimeout': '30'
- }
-
- if is_fifo:
- expected_attributes['ContentBasedDeduplication'] = 'false'
- expected_attributes['FifoQueue'] = 'true'
-
- attributes = self._sqs_api.get_queue_attributes(queue_url)
- for attr in expected_attributes:
- assert_that(attributes[attr], equal_to(expected_attributes[attr]))
-
- @pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_create_queue_with_custom_attributes(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
-
- custom_attributes = {
- 'DelaySeconds': '1',
- 'MaximumMessageSize': '262143',
- 'MessageRetentionPeriod': '345599',
- 'ReceiveMessageWaitTimeSeconds': '1',
- 'VisibilityTimeout': '29'
- }
-
- if is_fifo:
- custom_attributes['ContentBasedDeduplication'] = 'true'
-
- queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo, attributes=custom_attributes)
-
- attributes = self._sqs_api.get_queue_attributes(queue_url)
- for attr in custom_attributes:
- assert_that(attributes[attr], equal_to(custom_attributes[attr]))
-
- # get arn by default
+ assert('InvalidAttributeName' in str(e))
+ else:
+ assert_that(False) # expected InvalidAttributeName exception
+
+ @pytest.mark.parametrize(**IS_FIFO_PARAMS)
+ def test_create_queue_with_default_attributes(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+
+ # check default queue creation
+ queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo)
+
+ expected_attributes = {
+ 'DelaySeconds': '0',
+ 'MaximumMessageSize': '262144',
+ 'MessageRetentionPeriod': '345600',
+ 'ReceiveMessageWaitTimeSeconds': '0',
+ 'VisibilityTimeout': '30'
+ }
+
+ if is_fifo:
+ expected_attributes['ContentBasedDeduplication'] = 'false'
+ expected_attributes['FifoQueue'] = 'true'
+
+ attributes = self._sqs_api.get_queue_attributes(queue_url)
+ for attr in expected_attributes:
+ assert_that(attributes[attr], equal_to(expected_attributes[attr]))
+
+ @pytest.mark.parametrize(**IS_FIFO_PARAMS)
+ def test_create_queue_with_custom_attributes(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+
+ custom_attributes = {
+ 'DelaySeconds': '1',
+ 'MaximumMessageSize': '262143',
+ 'MessageRetentionPeriod': '345599',
+ 'ReceiveMessageWaitTimeSeconds': '1',
+ 'VisibilityTimeout': '29'
+ }
+
+ if is_fifo:
+ custom_attributes['ContentBasedDeduplication'] = 'true'
+
+ queue_url = self._sqs_api.create_queue(self.queue_name, is_fifo=is_fifo, attributes=custom_attributes)
+
+ attributes = self._sqs_api.get_queue_attributes(queue_url)
+ for attr in custom_attributes:
+ assert_that(attributes[attr], equal_to(custom_attributes[attr]))
+
+ # get arn by default
assert_that(attributes['QueueArn'], equal_to('yrn:ya:sqs:ru-central1:' + self._username + ':' + self.queue_name))
-
- # okay, now we'll try to break it
- for attr in custom_attributes:
- try:
- queue_url = self._sqs_api.create_queue('new_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2147483647'})
+
+ # okay, now we'll try to break it
+ for attr in custom_attributes:
+ try:
+ queue_url = self._sqs_api.create_queue('new_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2147483647'})
except Exception as e:
- assert(attr in str(e))
- else:
- assert_that(False) # expected some exception
-
- try:
- queue_url = self._sqs_api.create_queue('new2_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2.5'})
+ assert(attr in str(e))
+ else:
+ assert_that(False) # expected some exception
+
+ try:
+ queue_url = self._sqs_api.create_queue('new2_' + self.queue_name, is_fifo=is_fifo, attributes={attr: '2.5'})
except Exception as e:
- assert(attr in str(e))
- else:
- assert_that(False) # expected some exception
+ assert(attr in str(e))
+ else:
+ assert_that(False) # expected some exception
diff --git a/ydb/tests/functional/sqs/test_queues_managing.py b/ydb/tests/functional/sqs/test_queues_managing.py
index 58bf407ebf..bcfd91ca67 100644
--- a/ydb/tests/functional/sqs/test_queues_managing.py
+++ b/ydb/tests/functional/sqs/test_queues_managing.py
@@ -7,7 +7,7 @@ import pytest
from hamcrest import assert_that, equal_to, greater_than, not_none, none, has_item, has_items, raises, empty, instance_of
from sqs_matchers import ReadResponseMatcher
-
+
from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS
from sqs_test_base import to_bytes
from ydb import issues as ydb_issues
@@ -269,19 +269,19 @@ class QueuesManagingTest(KikimrSqsTestBase):
created_queue_url, 10, ReadResponseMatcher().with_message_ids([msg_id, ])
)
- def test_ya_count_queues(self):
- assert_that(self._sqs_api.private_count_queues(), equal_to('0'))
- q_url = self._create_queue_and_assert('new_q')
- self._create_queue_and_assert('new_q_2')
-
+ def test_ya_count_queues(self):
+ assert_that(self._sqs_api.private_count_queues(), equal_to('0'))
+ q_url = self._create_queue_and_assert('new_q')
+ self._create_queue_and_assert('new_q_2')
+
time.sleep(2.1)
- assert_that(self._sqs_api.private_count_queues(), equal_to('2'))
-
- self._sqs_api.delete_queue(q_url)
-
+ assert_that(self._sqs_api.private_count_queues(), equal_to('2'))
+
+ self._sqs_api.delete_queue(q_url)
+
time.sleep(2.1)
- assert_that(self._sqs_api.private_count_queues(), equal_to('1'))
-
+ assert_that(self._sqs_api.private_count_queues(), equal_to('1'))
+
def test_queues_count_over_limit(self):
urls = []
for i in range(10):
diff --git a/ydb/tests/functional/sqs/test_recompiles_requests.py b/ydb/tests/functional/sqs/test_recompiles_requests.py
index 3182bff9ad..367003323f 100644
--- a/ydb/tests/functional/sqs/test_recompiles_requests.py
+++ b/ydb/tests/functional/sqs/test_recompiles_requests.py
@@ -4,7 +4,7 @@ import pytest
from hamcrest import assert_that, not_none
from ydb.tests.library.common.types import Erasure
-
+
from sqs_test_base import KikimrSqsTestBase, IS_FIFO_PARAMS
diff --git a/ydb/tests/functional/sqs/ya.make b/ydb/tests/functional/sqs/ya.make
index f71e3a9b0f..bbb0a57bb8 100644
--- a/ydb/tests/functional/sqs/ya.make
+++ b/ydb/tests/functional/sqs/ya.make
@@ -52,8 +52,8 @@ PEERDIR(
ydb/tests/library
ydb/tests/library/sqs
contrib/python/xmltodict
- contrib/python/boto3
- contrib/python/botocore
+ contrib/python/boto3
+ contrib/python/botocore
)
FORK_SUBTESTS()