aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkomels <komels@yandex-team.ru>2022-02-10 16:50:21 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:50:21 +0300
commit0dd632d6fc5676c75d7004172992cefaa2192db0 (patch)
tree5d5cb817648f650d76cf1076100726fd9b8448e8
parent244466466f4f03621b0b399620f9094a16b511d2 (diff)
downloadydb-0dd632d6fc5676c75d7004172992cefaa2192db0.tar.gz
Restoring authorship annotation for <komels@yandex-team.ru>. Commit 2 of 2.
-rw-r--r--library/cpp/grpc/server/grpc_request.h8
-rw-r--r--library/cpp/grpc/server/grpc_request_base.h6
-rw-r--r--ydb/core/base/appdata.h8
-rw-r--r--ydb/core/client/server/msgbus_server_persqueue.cpp400
-rw-r--r--ydb/core/client/server/msgbus_server_persqueue.h66
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metacache.cpp924
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metacache.h248
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metarequest.cpp298
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metarequest.h46
-rw-r--r--ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp248
-rw-r--r--ydb/core/client/server/msgbus_server_proxy.cpp2
-rw-r--r--ydb/core/client/server/pq_metacache_v2_ut.cpp532
-rw-r--r--ydb/core/driver_lib/run/config.h2
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp34
-rw-r--r--ydb/core/driver_lib/run/run.cpp34
-rw-r--r--ydb/core/driver_lib/run/run.h2
-rw-r--r--ydb/core/persqueue/metering_typedefs.h36
-rw-r--r--ydb/core/persqueue/partition.cpp2
-rw-r--r--ydb/core/persqueue/pq_impl.cpp220
-rw-r--r--ydb/core/persqueue/pq_impl.h32
-rw-r--r--ydb/core/persqueue/pq_ut.cpp6
-rw-r--r--ydb/core/persqueue/pq_ut_slow.cpp2
-rw-r--r--ydb/core/persqueue/read_balancer.cpp2
-rw-r--r--ydb/core/protos/config.proto60
-rw-r--r--ydb/core/protos/msgbus_pq.proto2
-rw-r--r--ydb/core/protos/pqconfig.proto42
-rw-r--r--ydb/core/protos/services.proto2
-rw-r--r--ydb/core/protos/ya.make2
-rw-r--r--ydb/core/testlib/mock_pq_metacache.h70
-rw-r--r--ydb/core/testlib/test_client.cpp52
-rw-r--r--ydb/core/testlib/test_client.h10
-rw-r--r--ydb/core/testlib/test_pq_client.h434
-rw-r--r--ydb/core/tx/scheme_cache/scheme_cache.h8
-rw-r--r--ydb/core/ymq/actor/action.h140
-rw-r--r--ydb/core/ymq/actor/delete_message.cpp4
-rw-r--r--ydb/core/ymq/actor/events.h22
-rw-r--r--ydb/core/ymq/actor/index_events_processor.cpp982
-rw-r--r--ydb/core/ymq/actor/index_events_processor.h288
-rw-r--r--ydb/core/ymq/actor/proxy_actor.cpp22
-rw-r--r--ydb/core/ymq/actor/proxy_service.cpp4
-rw-r--r--ydb/core/ymq/actor/proxy_service.h4
-rw-r--r--ydb/core/ymq/actor/purge.cpp2
-rw-r--r--ydb/core/ymq/actor/queue_leader.cpp22
-rw-r--r--ydb/core/ymq/actor/queue_schema.cpp110
-rw-r--r--ydb/core/ymq/actor/receive_message.cpp10
-rw-r--r--ydb/core/ymq/actor/send_message.cpp6
-rw-r--r--ydb/core/ymq/actor/service.cpp174
-rw-r--r--ydb/core/ymq/actor/service.h34
-rw-r--r--ydb/core/ymq/actor/serviceid.h2
-rw-r--r--ydb/core/ymq/actor/ya.make2
-rw-r--r--ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp720
-rw-r--r--ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp40
-rw-r--r--ydb/core/ymq/actor/yc_search_ut/test_events_writer.h42
-rw-r--r--ydb/core/ymq/actor/yc_search_ut/ya.make38
-rw-r--r--ydb/core/ymq/base/action.cpp80
-rw-r--r--ydb/core/ymq/base/action.h6
-rw-r--r--ydb/core/ymq/base/counters.cpp586
-rw-r--r--ydb/core/ymq/base/counters.h214
-rw-r--r--ydb/core/ymq/base/ut/counters_ut.cpp204
-rw-r--r--ydb/core/ymq/http/http.cpp32
-rw-r--r--ydb/core/ymq/http/http.h10
-rw-r--r--ydb/library/persqueue/topic_parser/topic_parser.cpp150
-rw-r--r--ydb/library/persqueue/topic_parser/topic_parser.h682
-rw-r--r--ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp350
-rw-r--r--ydb/library/persqueue/topic_parser/ut/ya.make32
-rw-r--r--ydb/library/persqueue/topic_parser/ya.make8
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp212
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h204
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp12
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp58
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h16
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp2066
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h616
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h480
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp574
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp188
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp650
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp66
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h764
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h98
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h64
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h130
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make2
-rw-r--r--ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make6
-rw-r--r--ydb/public/sdk/python/ydb/connection.py8
-rw-r--r--ydb/services/datastreams/datastreams_ut.cpp206
-rw-r--r--ydb/services/datastreams/put_records_actor.h16
-rw-r--r--ydb/services/lib/actors/pq_schema_actor.cpp30
-rw-r--r--ydb/services/lib/actors/pq_schema_actor.h2
-rw-r--r--ydb/services/lib/actors/type_definitions.h78
-rw-r--r--ydb/services/persqueue_cluster_discovery/grpc_service.cpp32
-rw-r--r--ydb/services/persqueue_cluster_discovery/grpc_service.h6
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_actor.h108
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_read.cpp36
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_read.h6
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_read_actor.cpp552
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_schema.cpp4
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_write.cpp32
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_write.h6
-rw-r--r--ydb/services/persqueue_v1/grpc_pq_write_actor.cpp172
-rw-r--r--ydb/services/persqueue_v1/persqueue_common_ut.cpp132
-rw-r--r--ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp100
-rw-r--r--ydb/services/persqueue_v1/persqueue_ut.cpp986
-rw-r--r--ydb/services/persqueue_v1/persqueue_utils.cpp148
-rw-r--r--ydb/services/persqueue_v1/persqueue_utils.h48
-rw-r--r--ydb/services/persqueue_v1/ut/api_test_setup.h22
-rw-r--r--ydb/services/persqueue_v1/ut/persqueue_test_fixture.h164
-rw-r--r--ydb/services/persqueue_v1/ut/pq_data_writer.h12
-rw-r--r--ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp52
-rw-r--r--ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h10
-rw-r--r--ydb/services/persqueue_v1/ut/test_utils.h2
-rw-r--r--ydb/services/ydb/ydb_common_ut.h6
-rw-r--r--ydb/tests/functional/blobstorage/replication.py30
-rw-r--r--ydb/tests/functional/blobstorage/ya.make14
-rw-r--r--ydb/tests/functional/hive/test_kill_tablets.py6
-rw-r--r--ydb/tests/functional/hive/ya.make2
-rw-r--r--ydb/tests/functional/sqs/sqs_matchers.py290
-rw-r--r--ydb/tests/functional/sqs/sqs_requests_client.py218
-rw-r--r--ydb/tests/functional/sqs/sqs_test_base.py158
-rw-r--r--ydb/tests/functional/sqs/test_fifo_messaging.py306
-rw-r--r--ydb/tests/functional/sqs/test_garbage_collection.py2
-rw-r--r--ydb/tests/functional/sqs/test_generic_messaging.py246
-rw-r--r--ydb/tests/functional/sqs/test_multinode_cluster.py2
-rw-r--r--ydb/tests/functional/sqs/test_polling.py2
-rw-r--r--ydb/tests/functional/sqs/test_queue_attributes_validation.py2
-rw-r--r--ydb/tests/functional/sqs/test_queues_managing.py88
-rw-r--r--ydb/tests/functional/sqs/test_recompiles_requests.py2
-rw-r--r--ydb/tests/functional/sqs/ya.make2
-rw-r--r--ydb/tests/library/common/composite_assert.py50
-rw-r--r--ydb/tests/library/common/generators.py92
-rw-r--r--ydb/tests/library/common/protobuf_ss.py8
-rw-r--r--ydb/tests/library/common/yatest_common.py6
-rw-r--r--ydb/tests/library/harness/daemon.py2
-rw-r--r--ydb/tests/library/harness/kikimr_client.py8
-rw-r--r--ydb/tests/library/harness/kikimr_cluster.py6
-rw-r--r--ydb/tests/library/harness/kikimr_config.py2
-rw-r--r--ydb/tests/library/harness/kikimr_http_client.py86
-rw-r--r--ydb/tests/library/harness/kikimr_runner.py26
-rw-r--r--ydb/tests/library/matchers/response.py158
-rw-r--r--ydb/tests/library/matchers/tablets.py182
-rw-r--r--ydb/tests/library/nemesis/nemesis_core.py566
-rw-r--r--ydb/tests/library/nemesis/nemesis_process_killers.py244
-rw-r--r--ydb/tests/library/nemesis/nemesis_time_terrorist.py86
-rw-r--r--ydb/tests/library/nemesis/remote_execution.py34
-rw-r--r--ydb/tests/library/nemesis/safety_warden.py62
147 files changed, 10519 insertions, 10519 deletions
diff --git a/library/cpp/grpc/server/grpc_request.h b/library/cpp/grpc/server/grpc_request.h
index b61ac1a351..5bd8d3902b 100644
--- a/library/cpp/grpc/server/grpc_request.h
+++ b/library/cpp/grpc/server/grpc_request.h
@@ -113,10 +113,10 @@ public:
return FinishPromise_.GetFuture();
}
- TString GetPeer() const override {
- return TString(this->Context.peer());
- }
-
+ TString GetPeer() const override {
+ return TString(this->Context.peer());
+ }
+
bool SslServer() const override {
return Server_->SslServer();
}
diff --git a/library/cpp/grpc/server/grpc_request_base.h b/library/cpp/grpc/server/grpc_request_base.h
index c80c62cb60..fcfce1c181 100644
--- a/library/cpp/grpc/server/grpc_request_base.h
+++ b/library/cpp/grpc/server/grpc_request_base.h
@@ -105,9 +105,9 @@ public:
//! Returns future to get cancel of finish notification
virtual TAsyncFinishResult GetFinishFuture() = 0;
-
- //! Returns peer address
- virtual TString GetPeer() const = 0;
+
+ //! Returns peer address
+ virtual TString GetPeer() const = 0;
//! Returns true if server is using ssl
virtual bool SslServer() const = 0;
diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h
index 976e99eaab..c666f7468c 100644
--- a/ydb/core/base/appdata.h
+++ b/ydb/core/base/appdata.h
@@ -34,10 +34,10 @@ namespace NMonitoring {
class TBusNgMonPage;
}
-namespace NYdb {
- class TDriver;
-}
-
+namespace NYdb {
+ class TDriver;
+}
+
namespace NKikimr {
namespace NScheme {
diff --git a/ydb/core/client/server/msgbus_server_persqueue.cpp b/ydb/core/client/server/msgbus_server_persqueue.cpp
index f2e1d7ca83..78b83ef3a3 100644
--- a/ydb/core/client/server/msgbus_server_persqueue.cpp
+++ b/ydb/core/client/server/msgbus_server_persqueue.cpp
@@ -14,9 +14,9 @@
namespace NKikimr {
namespace NMsgBusProxy {
-using namespace NSchemeCache;
-using namespace NPqMetaCacheV2;
-
+using namespace NSchemeCache;
+using namespace NPqMetaCacheV2;
+
const TDuration TPersQueueBaseRequestProcessor::TIMEOUT = TDuration::MilliSeconds(90000);
namespace {
@@ -32,104 +32,104 @@ const TString& TopicPrefix(const TActorContext& ctx) {
return topicPrefix;
}
-TProcessingResult ProcessMetaCacheAllTopicsResponse(TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev) {
- auto *res = ev->Get()->Result.Get();
- const TString& path = ev->Get()->Path;
- TProcessingResult result;
- if (!ev->Get()->Success) {
- return TProcessingResult{
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- Sprintf("path '%s' has invalid/unknown root prefix, Marker# PQ14", path.c_str()),
- true
- };
- }
- if (!res) {
- return TProcessingResult{
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::ERROR,
- Sprintf("path '%s' describe error, Status# no status, reason: no reason, Marker# PQ1", path.c_str()),
- true
- };
- }
- return {};
-}
-
-TProcessingResult ProcessMetaCacheSingleTopicsResponse(
- const TSchemeCacheNavigate::TEntry& entry
-) {
- auto fullPath = JoinPath(entry.Path);
- switch (entry.Status) {
- case TSchemeCacheNavigate::EStatus::RootUnknown : {
- return TProcessingResult {
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14",
- fullPath.c_str(), entry.Path[0].c_str()),
- true
- };
- }
- case TSchemeCacheNavigate::EStatus::PathErrorUnknown: {
- return TProcessingResult {
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- Sprintf("no path '%s', Marker# PQ150", fullPath.c_str()),
- true
- };
- }
- case TSchemeCacheNavigate::EStatus::Ok:
- break;
- default: {
- return TProcessingResult {
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::ERROR,
- Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1",
- fullPath.c_str(), ToString(entry.Status).c_str()),
- true
- };
- }
- }
-
- if (entry.Kind != TSchemeCacheNavigate::KindTopic) {
- return TProcessingResult {
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()),
- true
- };
- }
- if (!entry.PQGroupInfo) {
- return TProcessingResult {
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ2",
- fullPath.c_str()),
- true
- };
- }
- auto topicName = entry.PQGroupInfo->Description.GetName();
- if (topicName.empty()) {
- topicName = entry.Path.back();
- }
- if (!entry.PQGroupInfo->Description.HasBalancerTabletID()) {
- return TProcessingResult {
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()),
- true
- };
- }
- if (entry.PQGroupInfo->Description.GetBalancerTabletID() == 0) {
- return TProcessingResult {
- MSTATUS_ERROR,
- NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- Sprintf("topic '%s' is not created, Marker# PQ94", topicName.c_str()),
- true
- };
- }
- return {};
-}
-
+TProcessingResult ProcessMetaCacheAllTopicsResponse(TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev) {
+ auto *res = ev->Get()->Result.Get();
+ const TString& path = ev->Get()->Path;
+ TProcessingResult result;
+ if (!ev->Get()->Success) {
+ return TProcessingResult{
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ Sprintf("path '%s' has invalid/unknown root prefix, Marker# PQ14", path.c_str()),
+ true
+ };
+ }
+ if (!res) {
+ return TProcessingResult{
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::ERROR,
+ Sprintf("path '%s' describe error, Status# no status, reason: no reason, Marker# PQ1", path.c_str()),
+ true
+ };
+ }
+ return {};
+}
+
+TProcessingResult ProcessMetaCacheSingleTopicsResponse(
+ const TSchemeCacheNavigate::TEntry& entry
+) {
+ auto fullPath = JoinPath(entry.Path);
+ switch (entry.Status) {
+ case TSchemeCacheNavigate::EStatus::RootUnknown : {
+ return TProcessingResult {
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14",
+ fullPath.c_str(), entry.Path[0].c_str()),
+ true
+ };
+ }
+ case TSchemeCacheNavigate::EStatus::PathErrorUnknown: {
+ return TProcessingResult {
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ Sprintf("no path '%s', Marker# PQ150", fullPath.c_str()),
+ true
+ };
+ }
+ case TSchemeCacheNavigate::EStatus::Ok:
+ break;
+ default: {
+ return TProcessingResult {
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::ERROR,
+ Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1",
+ fullPath.c_str(), ToString(entry.Status).c_str()),
+ true
+ };
+ }
+ }
+
+ if (entry.Kind != TSchemeCacheNavigate::KindTopic) {
+ return TProcessingResult {
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()),
+ true
+ };
+ }
+ if (!entry.PQGroupInfo) {
+ return TProcessingResult {
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ2",
+ fullPath.c_str()),
+ true
+ };
+ }
+ auto topicName = entry.PQGroupInfo->Description.GetName();
+ if (topicName.empty()) {
+ topicName = entry.Path.back();
+ }
+ if (!entry.PQGroupInfo->Description.HasBalancerTabletID()) {
+ return TProcessingResult {
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()),
+ true
+ };
+ }
+ if (entry.PQGroupInfo->Description.GetBalancerTabletID() == 0) {
+ return TProcessingResult {
+ MSTATUS_ERROR,
+ NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ Sprintf("topic '%s' is not created, Marker# PQ94", topicName.c_str()),
+ true
+ };
+ }
+ return {};
+}
+
NKikimrClient::TResponse CreateErrorReply(EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) {
NKikimrClient::TResponse rec;
rec.SetStatus(status);
@@ -143,10 +143,10 @@ struct TTopicInfo {
THashMap<ui32, ui64> PartitionToTablet;
ui64 BalancerTabletId = 0;
- THolder<NKikimrPQ::TReadSessionsInfoResponse> ReadSessionsInfo;
+ THolder<NKikimrPQ::TReadSessionsInfoResponse> ReadSessionsInfo;
NKikimrPQ::TPQTabletConfig Config;
- TIntrusiveConstPtr<TSchemeCacheNavigate::TPQGroupInfo> PQInfo;
+ TIntrusiveConstPtr<TSchemeCacheNavigate::TPQGroupInfo> PQInfo;
ui32 NumParts = 0;
THashSet<ui32> PartitionsToRequest;
@@ -164,10 +164,10 @@ struct TTabletInfo {
TVector<NKikimrPQ::TStatusResponse::TPartResult> StatusResponses;
};
-TPersQueueBaseRequestProcessor::TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes)
+TPersQueueBaseRequestProcessor::TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes)
: RequestProto(new NKikimrClient::TPersQueueRequest(request))
, RequestId(RequestProto->HasRequestId() ? RequestProto->GetRequestId() : "<none>")
- , PqMetaCache(pqMetaCacheId)
+ , PqMetaCache(pqMetaCacheId)
, ListNodes(listNodes)
{
}
@@ -192,7 +192,7 @@ void TPersQueueBaseRequestProcessor::Bootstrap(const TActorContext& ctx) {
LOG_INFO_S(ctx, NKikimrServices::PERSQUEUE, "proxy got request " << RequestId);
StartTimestamp = ctx.Now();
- ctx.Send(PqMetaCache, new NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest(TopicPrefix(ctx)));
+ ctx.Send(PqMetaCache, new NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest(TopicPrefix(ctx)));
if (ListNodes) {
const TActorId nameserviceId = GetNameserviceActorId();
@@ -213,8 +213,8 @@ void TPersQueueBaseRequestProcessor::Die(const TActorContext& ctx) {
STFUNC(TPersQueueBaseRequestProcessor::StateFunc) {
switch (ev->GetTypeRewrite()) {
HFunc(TEvInterconnect::TEvNodesInfo, Handle);
- HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse, Handle);
- HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse, Handle);
+ HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse, Handle);
+ HFunc(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse, Handle);
HFunc(TEvPersQueue::TEvResponse, Handle);
CFunc(TEvents::TSystem::Wakeup, HandleTimeout);
CFunc(NActors::TEvents::TSystem::PoisonPill, Die);
@@ -284,31 +284,31 @@ void TPersQueueBaseRequestProcessor::GetTopicsListOrThrow(const ::google::protob
throw std::runtime_error(desc);
}
}
-
+
+}
+
+void TPersQueueBaseRequestProcessor::Handle(
+ NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr&, const TActorContext&
+) {
+ Y_FAIL();
}
-void TPersQueueBaseRequestProcessor::Handle(
- NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr&, const TActorContext&
-) {
- Y_FAIL();
-}
-
-void TPersQueueBaseRequestProcessor::Handle(
- NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx
-) {
- auto& path = ev->Get()->Path;
- if (!ev->Get()->Success) {
+void TPersQueueBaseRequestProcessor::Handle(
+ NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx
+) {
+ auto& path = ev->Get()->Path;
+ if (!ev->Get()->Success) {
return SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC,
TStringBuilder() << "no path '" << path << "', Marker# PQ17");
}
- if (path != TopicPrefix(ctx)) {
- return SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- TStringBuilder() << "path '" << path << "' has no correct root prefix '" << TopicPrefix(ctx)
+ if (path != TopicPrefix(ctx)) {
+ return SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC,
+ TStringBuilder() << "path '" << path << "' has no correct root prefix '" << TopicPrefix(ctx)
<< "', Marker# PQ18");
}
- SchemeCacheResponse = std::move(ev->Get()->Result);
+ SchemeCacheResponse = std::move(ev->Get()->Result);
if (ReadyToCreateChildren()) {
if (CreateChildren(ctx)) {
return;
@@ -321,13 +321,13 @@ bool TPersQueueBaseRequestProcessor::ReadyToCreateChildren() const {
}
bool TPersQueueBaseRequestProcessor::CreateChildren(const TActorContext& ctx) {
- for (const auto& child : SchemeCacheResponse->ResultSet) {
- if (child.Kind == TSchemeCacheNavigate::EKind::KindTopic) {
- TString name = child.PQGroupInfo->Description.GetName();
- if (name.empty()) {
- name = child.Path.back();
- }
- if (!TopicsToRequest.empty() && !IsIn(TopicsToRequest, name)) {
+ for (const auto& child : SchemeCacheResponse->ResultSet) {
+ if (child.Kind == TSchemeCacheNavigate::EKind::KindTopic) {
+ TString name = child.PQGroupInfo->Description.GetName();
+ if (name.empty()) {
+ name = child.Path.back();
+ }
+ if (!TopicsToRequest.empty() && !IsIn(TopicsToRequest, name)) {
continue;
}
ChildrenToCreate.emplace_back(new TPerTopicInfo(child));
@@ -357,12 +357,12 @@ bool TPersQueueBaseRequestProcessor::CreateChildrenIfNeeded(const TActorContext&
while (!ChildrenToCreate.empty()) {
THolder<TPerTopicInfo> perTopicInfo(ChildrenToCreate.front().Release());
ChildrenToCreate.pop_front();
- const auto& name = perTopicInfo->TopicEntry.PQGroupInfo->Description.GetName();
- THolder<IActor> childActor = CreateTopicSubactor(perTopicInfo->TopicEntry, name);
+ const auto& name = perTopicInfo->TopicEntry.PQGroupInfo->Description.GetName();
+ THolder<IActor> childActor = CreateTopicSubactor(perTopicInfo->TopicEntry, name);
if (childActor.Get() != nullptr) {
const TActorId actorId = ctx.Register(childActor.Release());
perTopicInfo->ActorId = actorId;
- topics.emplace(name);
+ topics.emplace(name);
Children.emplace(actorId, std::move(perTopicInfo));
}
}
@@ -409,20 +409,20 @@ TPersQueueBaseRequestProcessor::TNodesInfo::TNodesInfo(THolder<TEvInterconnect::
}
}
-TTopicInfoBasedActor::TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName)
- : TActorBootstrapped<TTopicInfoBasedActor>()
- , SchemeEntry(topicEntry)
- , Name(topicName)
- , ProcessingResult(ProcessMetaCacheSingleTopicsResponse(SchemeEntry))
+TTopicInfoBasedActor::TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName)
+ : TActorBootstrapped<TTopicInfoBasedActor>()
+ , SchemeEntry(topicEntry)
+ , Name(topicName)
+ , ProcessingResult(ProcessMetaCacheSingleTopicsResponse(SchemeEntry))
{
}
-void TTopicInfoBasedActor::Bootstrap(const TActorContext &ctx) {
- Become(&TTopicInfoBasedActor::StateFunc);
- BootstrapImpl(ctx);
+void TTopicInfoBasedActor::Bootstrap(const TActorContext &ctx) {
+ Become(&TTopicInfoBasedActor::StateFunc);
+ BootstrapImpl(ctx);
}
-STFUNC(TTopicInfoBasedActor::StateFunc) {
+STFUNC(TTopicInfoBasedActor::StateFunc) {
switch (ev->GetTypeRewrite()) {
CFunc(NActors::TEvents::TSystem::PoisonPill, Die);
default:
@@ -431,15 +431,15 @@ STFUNC(TTopicInfoBasedActor::StateFunc) {
}
-class TMessageBusServerPersQueueImpl : public TActorBootstrapped<TMessageBusServerPersQueueImpl> {
- using TEvAllTopicsDescribeRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest;
- using TEvAllTopicsDescribeResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse;
+class TMessageBusServerPersQueueImpl : public TActorBootstrapped<TMessageBusServerPersQueueImpl> {
+ using TEvAllTopicsDescribeRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsRequest;
+ using TEvAllTopicsDescribeResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse;
protected:
NKikimrClient::TPersQueueRequest RequestProto;
const TString RequestId;
const bool IsMetaRequest;
- const bool IsFetchRequest;
+ const bool IsFetchRequest;
bool CanProcessFetchRequest; //any partitions answered that it has data or WaitMs timeout occured
ui32 FetchRequestReadsDone;
@@ -738,10 +738,10 @@ public:
THashMap<ui32, ui32> partitionToResp;
ui32 sz = 0;
- Y_VERIFY(p.second.ReadSessionsInfo);
- auto* sessionsInfo = p.second.ReadSessionsInfo.Get();
- for (ui32 i = 0; i < sessionsInfo->PartitionInfoSize(); ++i) {
- const auto& resp = sessionsInfo->GetPartitionInfo(i);
+ Y_VERIFY(p.second.ReadSessionsInfo);
+ auto* sessionsInfo = p.second.ReadSessionsInfo.Get();
+ for (ui32 i = 0; i < sessionsInfo->PartitionInfoSize(); ++i) {
+ const auto& resp = sessionsInfo->GetPartitionInfo(i);
partitionToResp[resp.GetPartition()] = sz++;
auto res = topicRes->AddPartitionResult();
res->SetPartition(resp.GetPartition());
@@ -846,7 +846,7 @@ public:
auto jt = TopicInfo.find(it->second.Topic);
Y_VERIFY(jt != TopicInfo.end());
- jt->second.ReadSessionsInfo = MakeHolder<NKikimrPQ::TReadSessionsInfoResponse>(std::move(response));
+ jt->second.ReadSessionsInfo = MakeHolder<NKikimrPQ::TReadSessionsInfoResponse>(std::move(response));
AnswerIfCanForMeta(ctx);
}
@@ -862,7 +862,7 @@ public:
it->second.StatusResponses.push_back(response.GetPartResult(i));
}
TabletsAnswered.insert(it->first);
-
+
AnswerIfCanForMeta(ctx);
}
@@ -872,37 +872,37 @@ public:
}
- void Handle(TEvAllTopicsDescribeResponse::TPtr& ev, const TActorContext& ctx) {
+ void Handle(TEvAllTopicsDescribeResponse::TPtr& ev, const TActorContext& ctx) {
--DescribeRequests;
- auto* res = ev->Get()->Result.Get();
- auto processResult = ProcessMetaCacheAllTopicsResponse(ev);
- if (processResult.IsFatal) {
- ErrorReason = processResult.Reason;
- return SendReplyAndDie(CreateErrorReply(processResult.Status, processResult.ErrorCode, ctx), ctx);
+ auto* res = ev->Get()->Result.Get();
+ auto processResult = ProcessMetaCacheAllTopicsResponse(ev);
+ if (processResult.IsFatal) {
+ ErrorReason = processResult.Reason;
+ return SendReplyAndDie(CreateErrorReply(processResult.Status, processResult.ErrorCode, ctx), ctx);
}
NoTopicsAtStart = TopicInfo.empty();
bool hasTopics = !NoTopicsAtStart;
- for (const auto& entry : res->ResultSet) {
- if (entry.Kind == TSchemeCacheNavigate::EKind::KindTopic && entry.PQGroupInfo) {
- auto& description = entry.PQGroupInfo->Description;
- if (!hasTopics || TopicInfo.find(description.GetName()) != TopicInfo.end()) {
- auto& topicInfo = TopicInfo[description.GetName()];
- topicInfo.BalancerTabletId = description.GetBalancerTabletID();
- topicInfo.PQInfo = entry.PQGroupInfo;
+ for (const auto& entry : res->ResultSet) {
+ if (entry.Kind == TSchemeCacheNavigate::EKind::KindTopic && entry.PQGroupInfo) {
+ auto& description = entry.PQGroupInfo->Description;
+ if (!hasTopics || TopicInfo.find(description.GetName()) != TopicInfo.end()) {
+ auto& topicInfo = TopicInfo[description.GetName()];
+ topicInfo.BalancerTabletId = description.GetBalancerTabletID();
+ topicInfo.PQInfo = entry.PQGroupInfo;
}
}
}
- for (auto& p: TopicInfo) {
+ for (auto& p: TopicInfo) {
const TString& topic = p.first;
if (!p.second.BalancerTabletId) {
ErrorReason = Sprintf("topic '%s' is not created, Marker# PQ94", topic.c_str());
return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx), ctx);
}
- ProcessMetadata(p.first, p.second, ctx);
+ ProcessMetadata(p.first, p.second, ctx);
}
if (RequestProto.HasMetaRequest()) {
@@ -910,22 +910,22 @@ public:
}
}
- void ProcessMetadata(const TString& name, TTopicInfo& info, const TActorContext& ctx) {
- //const TString& name = info.PQInfo->Description.GetName();
- if (!info.PQInfo) { //not supposed to happen anymore
+ void ProcessMetadata(const TString& name, TTopicInfo& info, const TActorContext& ctx) {
+ //const TString& name = info.PQInfo->Description.GetName();
+ if (!info.PQInfo) { //not supposed to happen anymore
if (RequestProto.HasMetaRequest() && NoTopicsAtStart && !RequestProto.GetMetaRequest().HasCmdGetTopicMetadata()) {
++TopicsAnswered;
AnswerIfCanForMeta(ctx);
} else {
- ErrorReason = Sprintf("topic '%s' is not ready, Marker# PQ85", name.c_str());
- SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx), ctx);
+ ErrorReason = Sprintf("topic '%s' is not ready, Marker# PQ85", name.c_str());
+ SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC, ctx), ctx);
}
return;
}
- const auto& pqDescr = info.PQInfo->Description;
- const TString& topic = pqDescr.GetName();
- Y_VERIFY(topic == name, "topic '%s' path '%s'", topic.c_str(), name.c_str());
+ const auto& pqDescr = info.PQInfo->Description;
+ const TString& topic = pqDescr.GetName();
+ Y_VERIFY(topic == name, "topic '%s' path '%s'", topic.c_str(), name.c_str());
bool mirrorerRequest = false;
if (RequestProto.HasPartitionRequest()) {
@@ -960,10 +960,10 @@ public:
}
if (AppData(ctx)->PQConfig.GetCheckACL() && operation && !mirrorerRequest) {
- if (*operation == NKikimrPQ::EOperation::WRITE_OP && pqDescr.GetPQTabletConfig().GetRequireAuthWrite() ||
- *operation == NKikimrPQ::EOperation::READ_OP && pqDescr.GetPQTabletConfig().GetRequireAuthRead()) {
+ if (*operation == NKikimrPQ::EOperation::WRITE_OP && pqDescr.GetPQTabletConfig().GetRequireAuthWrite() ||
+ *operation == NKikimrPQ::EOperation::READ_OP && pqDescr.GetPQTabletConfig().GetRequireAuthRead()) {
ErrorReason = Sprintf("unauthenticated access to '%s' is denied, Marker# PQ419", topic.c_str());
- return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::ACCESS_DENIED, ctx), ctx);
+ return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, NPersQueue::NErrorCode::ACCESS_DENIED, ctx), ctx);
}
}
@@ -973,9 +973,9 @@ public:
Y_VERIFY(it != TopicInfo.end());
Y_VERIFY(it->second.PartitionsToRequest.size() == 1);
ui32 partition = *(it->second.PartitionsToRequest.begin());
- for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) {
- const auto& pi = pqDescr.GetPartitions(i);
- if (pi.GetPartitionId() == partition) {
+ for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) {
+ const auto& pi = pqDescr.GetPartitions(i);
+ if (pi.GetPartitionId() == partition) {
tabletId = pi.GetTabletId();
break;
}
@@ -1011,9 +1011,9 @@ public:
++TopicsAnswered;
auto it = TopicInfo.find(topic);
Y_VERIFY(it != TopicInfo.end(), "topic '%s'", topic.c_str());
- it->second.Config = pqDescr.GetPQTabletConfig();
- it->second.Config.SetVersion(pqDescr.GetAlterVersion());
- it->second.NumParts = pqDescr.PartitionsSize();
+ it->second.Config = pqDescr.GetPQTabletConfig();
+ it->second.Config.SetVersion(pqDescr.GetAlterVersion());
+ it->second.NumParts = pqDescr.PartitionsSize();
if (metadataOnly) {
AnswerIfCanForMeta(ctx);
return;
@@ -1041,9 +1041,9 @@ public:
NTabletPipe::SendData(ctx, pipeClient, ev.Release());
}
- for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) {
- ui32 part = pqDescr.GetPartitions(i).GetPartitionId();
- ui64 tabletId = pqDescr.GetPartitions(i).GetTabletId();
+ for (ui32 i = 0; i < pqDescr.PartitionsSize(); ++i) {
+ ui32 part = pqDescr.GetPartitions(i).GetPartitionId();
+ ui64 tabletId = pqDescr.GetPartitions(i).GetTabletId();
if (!it->second.PartitionsToRequest.empty() && !it->second.PartitionsToRequest.contains(part)) {
continue;
}
@@ -1152,14 +1152,14 @@ public:
ui32 nodeId = ev->Get()->ServerId.NodeId();
it->second.NodeId = nodeId;
TabletsDiscovered.insert(tabletId);
-
+
AnswerIfCanForMeta(ctx);
}
}
}
void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) {
-
+
ui64 tabletId = ev->Get()->TabletId;
if (HandlePipeError(tabletId, ctx))
return;
@@ -1310,8 +1310,8 @@ public:
ctx.Schedule(TDuration::MilliSeconds(Min<ui32>(RequestProto.GetFetchRequest().GetWaitMs(), 30000)), new TEvPersQueue::TEvHasDataInfoResponse);
}
- auto* request = new TEvAllTopicsDescribeRequest(TopicPrefix(ctx));
- ctx.Send(SchemeCache, request);
+ auto* request = new TEvAllTopicsDescribeRequest(TopicPrefix(ctx));
+ ctx.Send(SchemeCache, request);
++DescribeRequests;
if (RequestProto.HasMetaRequest() && (RequestProto.GetMetaRequest().HasCmdGetPartitionLocations()
@@ -1324,9 +1324,9 @@ public:
Become(&TMessageBusServerPersQueueImpl::StateFunc, ctx, TDuration::MilliSeconds(DefaultTimeout), new TEvents::TEvWakeup());
}
- STRICT_STFUNC(StateFunc,
+ STRICT_STFUNC(StateFunc,
HFunc(TEvInterconnect::TEvNodesInfo, Handle);
- HFunc(TEvAllTopicsDescribeResponse, Handle);
+ HFunc(TEvAllTopicsDescribeResponse, Handle);
HFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
HFunc(TEvTabletPipe::TEvClientConnected, Handle);
HFunc(TEvPersQueue::TEvResponse, Handle);
@@ -1336,7 +1336,7 @@ public:
HFunc(TEvPersQueue::TEvReadSessionsInfoResponse, Handle);
CFunc(TEvents::TSystem::Wakeup, HandleTimeout);
CFunc(NActors::TEvents::TSystem::PoisonPill, Die);
- )
+ )
private:
bool GetTopicsList(const ::google::protobuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& requests) {
for (auto ri = requests.begin(); ri != requests.end(); ++ri) {
@@ -1355,12 +1355,12 @@ private:
return false;
}
}
- const auto& topic = topicRequest.GetTopic();
- if (TopicInfo.contains(topic)) {
- ErrorReason = Sprintf("multiple TopicRequest for topic '%s'", topic.c_str());
+ const auto& topic = topicRequest.GetTopic();
+ if (TopicInfo.contains(topic)) {
+ ErrorReason = Sprintf("multiple TopicRequest for topic '%s'", topic.c_str());
return false;
- } else {
- TopicInfo[topic] = std::move(topicInfo);
+ } else {
+ TopicInfo[topic] = std::move(topicInfo);
}
}
return true;
diff --git a/ydb/core/client/server/msgbus_server_persqueue.h b/ydb/core/client/server/msgbus_server_persqueue.h
index b74e51712e..52c67832e4 100644
--- a/ydb/core/client/server/msgbus_server_persqueue.h
+++ b/ydb/core/client/server/msgbus_server_persqueue.h
@@ -17,16 +17,16 @@ namespace NMsgBusProxy {
const TString& TopicPrefix(const TActorContext& ctx);
-struct TProcessingResult {
- EResponseStatus Status = MSTATUS_OK;
- NPersQueue::NErrorCode::EErrorCode ErrorCode;
- TString Reason;
- bool IsFatal = false;
-};
-
-TProcessingResult ProcessMetaCacheAllTopicsResponse(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& response);
-TProcessingResult ProcessMetaCacheSingleTopicsResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry);
-
+struct TProcessingResult {
+ EResponseStatus Status = MSTATUS_OK;
+ NPersQueue::NErrorCode::EErrorCode ErrorCode;
+ TString Reason;
+ bool IsFatal = false;
+};
+
+TProcessingResult ProcessMetaCacheAllTopicsResponse(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& response);
+TProcessingResult ProcessMetaCacheSingleTopicsResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry);
+
// Worker actor creation
IActor* CreateMessageBusServerPersQueue(
TBusMessageContext& msg,
@@ -60,20 +60,20 @@ inline ui64 GetTabletId<TEvTabletPipe::TEvClientConnected>(const TEvTabletPipe::
// 2. Implement CreateTopicSubactor() and, optionally, MergeSubactorReplies() methods.
class TPersQueueBaseRequestProcessor : public TActorBootstrapped<TPersQueueBaseRequestProcessor> {
protected:
- using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry;
- using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>;
- using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus;
-
+ using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry;
+ using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>;
+ using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus;
+
struct TPerTopicInfo {
TPerTopicInfo()
{ }
- explicit TPerTopicInfo(const TSchemeEntry& topicEntry)
+ explicit TPerTopicInfo(const TSchemeEntry& topicEntry)
: TopicEntry(topicEntry)
{
}
TActorId ActorId;
- TSchemeEntry TopicEntry;
+ TSchemeEntry TopicEntry;
NKikimrClient::TResponse Response;
bool ActorAnswered = false;
};
@@ -101,7 +101,7 @@ public:
}
protected:
- TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes);
+ TPersQueueBaseRequestProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& pqMetaCacheId, bool listNodes);
~TPersQueueBaseRequestProcessor();
@@ -111,7 +111,7 @@ public:
protected:
bool CreateChildrenIfNeeded(const TActorContext& ctx);
- virtual THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) = 0; // Creates actor for processing one concrete topic.
+ virtual THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) = 0; // Creates actor for processing one concrete topic.
virtual NKikimrClient::TResponse MergeSubactorReplies();
virtual void SendReplyAndDie(NKikimrClient::TResponse&& record, const TActorContext& ctx) = 0;
@@ -129,8 +129,8 @@ protected:
virtual STFUNC(StateFunc);
void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev, const TActorContext& ctx);
- void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx);
- void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx);
+ void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx);
+ void Handle(NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeAllTopicsResponse::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx);
void HandleTimeout(const TActorContext& ctx);
@@ -142,10 +142,10 @@ protected:
const TString RequestId;
THashSet<TString> TopicsToRequest; // Topics that we need to request. If this set id empty, we are interested in all existing topics.
- const TActorId PqMetaCache;
+ const TActorId PqMetaCache;
THashMap<TActorId, THolder<TPerTopicInfo>> Children;
size_t ChildrenAnswered = 0;
- THolder<NSchemeCache::TSchemeCacheNavigate> SchemeCacheResponse;
+ THolder<NSchemeCache::TSchemeCacheNavigate> SchemeCacheResponse;
// Nodes info
const bool ListNodes;
@@ -153,15 +153,15 @@ protected:
};
// Helper actor that sends TEvGetBalancerDescribe and checks ACL (ACL is not implemented yet).
-class TTopicInfoBasedActor : public TActorBootstrapped<TTopicInfoBasedActor> {
+class TTopicInfoBasedActor : public TActorBootstrapped<TTopicInfoBasedActor> {
protected:
- using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry;
- using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>;
- using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus;
+ using TSchemeEntry = NSchemeCache::TSchemeCacheNavigate::TEntry;
+ using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>;
+ using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus;
- TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName);
+ TTopicInfoBasedActor(const TSchemeEntry& topicEntry, const TString& topicName);
- virtual void BootstrapImpl(const TActorContext& ctx) = 0;
+ virtual void BootstrapImpl(const TActorContext& ctx) = 0;
virtual void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) = 0;
virtual void SendReplyAndDie(NKikimrClient::TResponse&& record, const TActorContext& ctx) = 0;
@@ -170,11 +170,11 @@ protected:
protected:
TActorId SchemeCache;
- TSchemeEntry SchemeEntry;
- TString Name;
- TProcessingResult ProcessingResult;
-public:
- void Bootstrap(const TActorContext& ctx);
+ TSchemeEntry SchemeEntry;
+ TString Name;
+ TProcessingResult ProcessingResult;
+public:
+ void Bootstrap(const TActorContext& ctx);
static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
return NKikimrServices::TActivity::PQ_BASE_REQUEST_PROCESSOR;
diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.cpp b/ydb/core/client/server/msgbus_server_pq_metacache.cpp
index ae3ac12f10..f44ab6f480 100644
--- a/ydb/core/client/server/msgbus_server_pq_metacache.cpp
+++ b/ydb/core/client/server/msgbus_server_pq_metacache.cpp
@@ -1,5 +1,5 @@
#include "msgbus_server_persqueue.h"
-#include "msgbus_server_pq_metacache.h"
+#include "msgbus_server_pq_metacache.h"
#include <ydb/public/api/protos/draft/persqueue_error_codes.pb.h>
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
@@ -7,89 +7,89 @@
#include <ydb/core/tx/schemeshard/schemeshard.h>
#include <ydb/core/tx/scheme_board/cache.h>
#include <ydb/core/base/counters.h>
-
+
#include <ydb/core/base/appdata.h>
-namespace NKikimr::NMsgBusProxy {
+namespace NKikimr::NMsgBusProxy {
-using namespace NYdb::NTable;
+using namespace NYdb::NTable;
-namespace NPqMetaCacheV2 {
+namespace NPqMetaCacheV2 {
-IActor* CreateSchemeCache(NActors::TActorSystem* ActorSystem, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) {
- auto appData = ActorSystem->AppData<TAppData>();
+IActor* CreateSchemeCache(NActors::TActorSystem* ActorSystem, TIntrusivePtr<NMonitoring::TDynamicCounters> counters) {
+ auto appData = ActorSystem->AppData<TAppData>();
auto cacheCounters = GetServiceCounters(counters, "pqproxy|schemecache");
auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(appData, cacheCounters);
- return CreateSchemeBoardSchemeCache(cacheConfig.Get());
-}
+ return CreateSchemeBoardSchemeCache(cacheConfig.Get());
+}
-class TPersQueueMetaCacheActor : public TActorBootstrapped<TPersQueueMetaCacheActor> {
- using TBase = TActorBootstrapped<TPersQueueMetaCacheActor>;
-public:
+class TPersQueueMetaCacheActor : public TActorBootstrapped<TPersQueueMetaCacheActor> {
+ using TBase = TActorBootstrapped<TPersQueueMetaCacheActor>;
+public:
TPersQueueMetaCacheActor(TPersQueueMetaCacheActor&&) = default;
TPersQueueMetaCacheActor& operator=(TPersQueueMetaCacheActor&&) = default;
TPersQueueMetaCacheActor(ui64 grpcPort,
const NMonitoring::TDynamicCounterPtr& counters,
const TDuration& versionCheckInterval)
- : Counters(counters)
+ : Counters(counters)
, ClientWrapper(std::move(std::make_unique<TClientWrapper>(grpcPort)))
- , VersionCheckInterval(versionCheckInterval)
+ , VersionCheckInterval(versionCheckInterval)
, Generation(std::make_shared<TAtomicCounter>())
- {
- }
-
+ {
+ }
+
TPersQueueMetaCacheActor(const NMonitoring::TDynamicCounterPtr& counters,
const TDuration& versionCheckInterval)
- : Counters(counters)
- , VersionCheckInterval(versionCheckInterval)
+ : Counters(counters)
+ , VersionCheckInterval(versionCheckInterval)
, Generation(std::make_shared<TAtomicCounter>())
{
}
- void Bootstrap(const TActorContext& ctx) {
- if (ClientWrapper == nullptr) {
+ void Bootstrap(const TActorContext& ctx) {
+ if (ClientWrapper == nullptr) {
auto* driver = AppData(ctx)->YdbDriver;
- if (driver == nullptr) {
- LOG_WARN_S(
- ctx, NKikimrServices::PQ_METACACHE,
- "Initialized without valid YDB driver - suppose misconfiguration. Refuse to work. Die."
- );
- Die(ctx);
- return;
- }
+ if (driver == nullptr) {
+ LOG_WARN_S(
+ ctx, NKikimrServices::PQ_METACACHE,
+ "Initialized without valid YDB driver - suppose misconfiguration. Refuse to work. Die."
+ );
+ Die(ctx);
+ return;
+ }
ClientWrapper.reset(new TClientWrapper(driver));
- }
- SkipVersionCheck = !AppData(ctx)->PQConfig.GetMetaCacheSkipVersionCheck();
- PathPrefix = TopicPrefix(ctx);
- TopicsQuery = TStringBuilder() << "--!syntax_v1\n"
- << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; "
- << "SELECT path, dc from `" << PathPrefix << "Config/V2/Topics` "
- << "WHERE path > $Path OR (path = $Path AND dc > $Cluster);";
-
- VersionQuery = TStringBuilder() << "--!syntax_v1\nSELECT version FROM `" << PathPrefix << "Config/V2/Versions` "
- << "WHERE name = 'Topics';";
- PathPrefixParts = NKikimr::SplitPath(PathPrefix);
- Become(&TPersQueueMetaCacheActor::StateFunc);
-
- if (!SchemeCacheId) {
- SchemeCacheId = Register(CreateSchemeCache(GetActorSystem(), Counters));
- }
- ActorSystem = GetActorSystem();
- Reset(ctx);
- }
-
- ~TPersQueueMetaCacheActor() {
- Generation->Inc();
- if (ClientWrapper)
- ClientWrapper->Stop();
- }
-
-private:
+ }
+ SkipVersionCheck = !AppData(ctx)->PQConfig.GetMetaCacheSkipVersionCheck();
+ PathPrefix = TopicPrefix(ctx);
+ TopicsQuery = TStringBuilder() << "--!syntax_v1\n"
+ << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; "
+ << "SELECT path, dc from `" << PathPrefix << "Config/V2/Topics` "
+ << "WHERE path > $Path OR (path = $Path AND dc > $Cluster);";
+
+ VersionQuery = TStringBuilder() << "--!syntax_v1\nSELECT version FROM `" << PathPrefix << "Config/V2/Versions` "
+ << "WHERE name = 'Topics';";
+ PathPrefixParts = NKikimr::SplitPath(PathPrefix);
+ Become(&TPersQueueMetaCacheActor::StateFunc);
+
+ if (!SchemeCacheId) {
+ SchemeCacheId = Register(CreateSchemeCache(GetActorSystem(), Counters));
+ }
+ ActorSystem = GetActorSystem();
+ Reset(ctx);
+ }
+
+ ~TPersQueueMetaCacheActor() {
+ Generation->Inc();
+ if (ClientWrapper)
+ ClientWrapper->Stop();
+ }
+
+private:
template<class TEventType, class TFutureType, class... TArgs>
void SubscribeEvent(const TFutureType& future, TArgs... args) {
std::weak_ptr<TAtomicCounter> weakGeneration(Generation);
- future.Subscribe(
+ future.Subscribe(
[
id = SelfId(),
originalGen = Generation->Val(),
@@ -100,235 +100,235 @@ private:
auto currentGen = weakGen.lock();
if (currentGen && originalGen == currentGen->Val()) {
as->Send(id, new TEventType(args...));
- }
+ }
}
- );
- }
-
- void Reset(const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Metacache: reset");
- Y_VERIFY(ClientWrapper);
- ClientWrapper->Reset();
- Generation->Inc();
- YdbSession = Nothing();
- PreparedTopicsQuery = Nothing();
- LastTopicKey = {};
- ctx.Schedule(QueryRetryInterval, new NActors::TEvents::TEvWakeup());
- }
-
- void StartSession(const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start new session");
- SessionFuture = ClientWrapper->GetClient()->GetSession();
+ );
+ }
+
+ void Reset(const TActorContext& ctx) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Metacache: reset");
+ Y_VERIFY(ClientWrapper);
+ ClientWrapper->Reset();
+ Generation->Inc();
+ YdbSession = Nothing();
+ PreparedTopicsQuery = Nothing();
+ LastTopicKey = {};
+ ctx.Schedule(QueryRetryInterval, new NActors::TEvents::TEvWakeup());
+ }
+
+ void StartSession(const TActorContext& ctx) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start new session");
+ SessionFuture = ClientWrapper->GetClient()->GetSession();
SubscribeEvent<TEvPqNewMetaCache::TEvSessionStarted>(SessionFuture);
- }
-
- void HandleSessionStarted(const TActorContext& ctx) {
- auto& value = SessionFuture.GetValue();
- if (!value.IsSuccess()) {
- LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, "Session start failed: " << value.GetIssues().ToString());
- return Reset(ctx);
- }
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Session started");
- YdbSession = value.GetSession();
- PrepareTopicsQuery(ctx);
- }
-
- void PrepareTopicsQuery(const TActorContext& ctx) {
- Y_UNUSED(ctx);
- TopicsQueryFuture = YdbSession->PrepareDataQuery(TopicsQuery);
+ }
+
+ void HandleSessionStarted(const TActorContext& ctx) {
+ auto& value = SessionFuture.GetValue();
+ if (!value.IsSuccess()) {
+ LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE, "Session start failed: " << value.GetIssues().ToString());
+ return Reset(ctx);
+ }
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Session started");
+ YdbSession = value.GetSession();
+ PrepareTopicsQuery(ctx);
+ }
+
+ void PrepareTopicsQuery(const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ TopicsQueryFuture = YdbSession->PrepareDataQuery(TopicsQuery);
SubscribeEvent<TEvPqNewMetaCache::TEvQueryPrepared>(TopicsQueryFuture);
- }
-
- void HandleTopicsQueryPrepared(const TActorContext& ctx) {
- auto& value = TopicsQueryFuture.GetValue();
- if (!value.IsSuccess()) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Topics query prepare failed: " << value.GetIssues().ToString());
- return Reset(ctx);
- }
- PreparedTopicsQuery = value.GetQuery();
- if (NewTopicsVersion > CurrentTopicsVersion) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start topics rescan");
- RunQuery(EQueryType::EGetTopics, ctx);
- } else {
- Y_VERIFY(NewTopicsVersion == CurrentTopicsVersion);
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Check version");
- RunQuery(EQueryType::ECheckVersion, ctx);
- }
- }
-
- void RunQuery(EQueryType type, const TActorContext& ctx) {
- Y_UNUSED(ctx);
- Y_VERIFY(YdbSession);
- if (type == EQueryType::ECheckVersion) {
- AsyncQueryResult = YdbSession->ExecuteDataQuery(VersionQuery, TTxControl::BeginTx().CommitTx());
- } else {
- Y_VERIFY(PreparedTopicsQuery);
- auto builder = PreparedTopicsQuery->GetParamsBuilder();
- {
- auto& param = builder.AddParam("$Path");
- param.Utf8(LastTopicKey.Path);
- param.Build();
- }
- {
- auto& param = builder.AddParam("$Cluster");
- param.Utf8(LastTopicKey.Cluster);
- param.Build();
- }
- AsyncQueryResult = PreparedTopicsQuery->Execute(TTxControl::BeginTx().CommitTx(), builder.Build());
- }
+ }
+
+ void HandleTopicsQueryPrepared(const TActorContext& ctx) {
+ auto& value = TopicsQueryFuture.GetValue();
+ if (!value.IsSuccess()) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Topics query prepare failed: " << value.GetIssues().ToString());
+ return Reset(ctx);
+ }
+ PreparedTopicsQuery = value.GetQuery();
+ if (NewTopicsVersion > CurrentTopicsVersion) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Start topics rescan");
+ RunQuery(EQueryType::EGetTopics, ctx);
+ } else {
+ Y_VERIFY(NewTopicsVersion == CurrentTopicsVersion);
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Check version");
+ RunQuery(EQueryType::ECheckVersion, ctx);
+ }
+ }
+
+ void RunQuery(EQueryType type, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ Y_VERIFY(YdbSession);
+ if (type == EQueryType::ECheckVersion) {
+ AsyncQueryResult = YdbSession->ExecuteDataQuery(VersionQuery, TTxControl::BeginTx().CommitTx());
+ } else {
+ Y_VERIFY(PreparedTopicsQuery);
+ auto builder = PreparedTopicsQuery->GetParamsBuilder();
+ {
+ auto& param = builder.AddParam("$Path");
+ param.Utf8(LastTopicKey.Path);
+ param.Build();
+ }
+ {
+ auto& param = builder.AddParam("$Cluster");
+ param.Utf8(LastTopicKey.Cluster);
+ param.Build();
+ }
+ AsyncQueryResult = PreparedTopicsQuery->Execute(TTxControl::BeginTx().CommitTx(), builder.Build());
+ }
SubscribeEvent<TEvPqNewMetaCache::TEvQueryComplete>(AsyncQueryResult, type);
- }
-
- void HandleCheckVersionResult(const TActorContext& ctx) {
- auto result = AsyncQueryResult.GetValue();
- if (!result.IsSuccess()) {
- LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE,
- "Got error trying to update config version: " << result.GetIssues().ToString());
- ctx.Schedule(QueryRetryInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion));
- return;
- }
- Y_VERIFY(result.GetResultSets().size() == 1);
- ui64 newVersion = 0;
- {
- auto versionQueryResult = result.GetResultSetParser(0);
- while (versionQueryResult.TryNextRow()) {
- newVersion = versionQueryResult.ColumnParser("version").GetOptionalInt64().GetOrElse(0);
- }
- }
- LastVersionUpdate = ctx.Now();
- if (newVersion > CurrentTopicsVersion || CurrentTopicsVersion == 0 || SkipVersionCheck) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Got config version: " << newVersion);
- NewTopicsVersion = newVersion;
- RunQuery(EQueryType::EGetTopics, ctx);
- } else {
- ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion));
- }
- }
-
- void HandleGetTopicsResult(const TActorContext& ctx) {
- auto result = AsyncQueryResult.GetValue();
- if (!result.IsSuccess()) {
- LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE,
- "Got error trying to get topics: " << result.GetIssues().ToString());
- return Reset(ctx);
- }
- Y_VERIFY(result.GetResultSets().size() == 1);
- auto versionQueryResult = result.GetResultSetParser(0);
- TString path, dc;
- while (versionQueryResult.TryNextRow()) {
- path = *versionQueryResult.ColumnParser("path").GetOptionalUtf8();
- dc = *versionQueryResult.ColumnParser("dc").GetOptionalUtf8();
- NewTopics.emplace_back(NPersQueue::BuildFullTopicName(path, dc));
- }
- if (result.GetResultSet(0).Truncated()) {
- LastTopicKey = {path, dc};
- return RunQuery(EQueryType::EGetTopics, ctx);
- } else {
- LastTopicKey = {};
- CurrentTopics = std::move(NewTopics);
- NewTopics.clear();
- EverGotTopics = true;
- CurrentTopicsVersion = NewTopicsVersion;
- FullTopicsCacheOutdated = true;
- FullTopicsCache = nullptr;
- while (!ListTopicsWaiters.empty()) {
- auto& waiter = ListTopicsWaiters.front();
- switch (waiter.Type) {
- case EWaiterType::ListTopics:
- SendListTopicsResponse(waiter.WaiterId, ctx);
- break;
- case EWaiterType::DescribeAllTopics:
- ProcessDescribeAllTopics(waiter.WaiterId, ctx);
- break;
- default:
- Y_FAIL();
- break;
- }
- ListTopicsWaiters.pop_front();
- }
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE,
- "Updated topics list with : " << CurrentTopics.size() << " topics");
- ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion));
- }
- }
-
- void HandleQueryComplete(TEvPqNewMetaCache::TEvQueryComplete::TPtr& ev, const TActorContext& ctx) {
- switch (ev->Get()->Type) {
- case EQueryType::ECheckVersion:
- return HandleCheckVersionResult(ctx);
- case EQueryType::EGetTopics:
- return HandleGetTopicsResult(ctx);
- default:
- Y_FAIL();
- }
- }
-
- void HandleRestartQuery(TEvPqNewMetaCache::TEvRestartQuery::TPtr& ev, const TActorContext& ctx) {
- Y_VERIFY(ev->Get()->Type == EQueryType::ECheckVersion);
- RunQuery(ev->Get()->Type, ctx);
- }
-
- void HandleListTopics(TEvPqNewMetaCache::TEvListTopicsRequest::TPtr& ev, const TActorContext& ctx) {
- if (!LastVersionUpdate || !EverGotTopics) {
- ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::ListTopics});
- } else {
- SendListTopicsResponse(ev->Sender, ctx);
- }
- }
-
- void HandleDescribeTopics(TEvPqNewMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Handle describe topics with prefix: " << ev->Get()->PathPrefix);
-
- SendSchemeCacheRequest(ev->Get()->Topics, !ev->Get()->PathPrefix.empty(), ctx);
- auto inserted = DescribeTopicsWaiters.insert(std::make_pair(
- RequestId,
- TWaiter{ev->Sender, std::move(ev->Get()->Topics), EWaiterType::DescribeCustomTopics}
- )).second;
- Y_VERIFY(inserted);
- }
-
- void HandleDescribeAllTopics(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) {
- if (ev->Get()->PathPrefix && ev->Get()->PathPrefix != PathPrefix) {
- auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse(ev->Get()->PathPrefix);
- response->Success = false;
- ctx.Send(ev->Sender, response);
- return;
- }
- if (!EverGotTopics) {
- ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::DescribeAllTopics});
- return;
- }
- return ProcessDescribeAllTopics(ev->Sender, ctx);
- }
-
- void ProcessDescribeAllTopics(const TActorId& waiterId, const TActorContext& ctx) {
- if (EverGotTopics && CurrentTopics.empty()) {
- SendDescribeAllTopicsResponse(waiterId, ctx, true);
- return;
- }
- if (FullTopicsCache && !FullTopicsCacheOutdated) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Respond from cache");
-
- return SendDescribeAllTopicsResponse(waiterId, ctx);
- }
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Store waiter");
- SendSchemeCacheRequest(CurrentTopics, true, ctx);
- auto inserted = DescribeTopicsWaiters.insert(std::make_pair(
- RequestId,
- TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics}
- )).second;
- Y_VERIFY(inserted);
- FullTopicsCacheOutdated = false;
- FullTopicsCache = nullptr;
- }
-
- void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
- auto* result = ev->Get()->Request.Release();
- auto waiterIter = DescribeTopicsWaiters.find(result->Instant);
- Y_VERIFY(!waiterIter.IsEnd());
- auto& waiter = waiterIter->second;
-
- if (waiter.Type == EWaiterType::DescribeAllTopics) {
+ }
+
+ void HandleCheckVersionResult(const TActorContext& ctx) {
+ auto result = AsyncQueryResult.GetValue();
+ if (!result.IsSuccess()) {
+ LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE,
+ "Got error trying to update config version: " << result.GetIssues().ToString());
+ ctx.Schedule(QueryRetryInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion));
+ return;
+ }
+ Y_VERIFY(result.GetResultSets().size() == 1);
+ ui64 newVersion = 0;
+ {
+ auto versionQueryResult = result.GetResultSetParser(0);
+ while (versionQueryResult.TryNextRow()) {
+ newVersion = versionQueryResult.ColumnParser("version").GetOptionalInt64().GetOrElse(0);
+ }
+ }
+ LastVersionUpdate = ctx.Now();
+ if (newVersion > CurrentTopicsVersion || CurrentTopicsVersion == 0 || SkipVersionCheck) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Got config version: " << newVersion);
+ NewTopicsVersion = newVersion;
+ RunQuery(EQueryType::EGetTopics, ctx);
+ } else {
+ ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion));
+ }
+ }
+
+ void HandleGetTopicsResult(const TActorContext& ctx) {
+ auto result = AsyncQueryResult.GetValue();
+ if (!result.IsSuccess()) {
+ LOG_ERROR_S(ctx, NKikimrServices::PQ_METACACHE,
+ "Got error trying to get topics: " << result.GetIssues().ToString());
+ return Reset(ctx);
+ }
+ Y_VERIFY(result.GetResultSets().size() == 1);
+ auto versionQueryResult = result.GetResultSetParser(0);
+ TString path, dc;
+ while (versionQueryResult.TryNextRow()) {
+ path = *versionQueryResult.ColumnParser("path").GetOptionalUtf8();
+ dc = *versionQueryResult.ColumnParser("dc").GetOptionalUtf8();
+ NewTopics.emplace_back(NPersQueue::BuildFullTopicName(path, dc));
+ }
+ if (result.GetResultSet(0).Truncated()) {
+ LastTopicKey = {path, dc};
+ return RunQuery(EQueryType::EGetTopics, ctx);
+ } else {
+ LastTopicKey = {};
+ CurrentTopics = std::move(NewTopics);
+ NewTopics.clear();
+ EverGotTopics = true;
+ CurrentTopicsVersion = NewTopicsVersion;
+ FullTopicsCacheOutdated = true;
+ FullTopicsCache = nullptr;
+ while (!ListTopicsWaiters.empty()) {
+ auto& waiter = ListTopicsWaiters.front();
+ switch (waiter.Type) {
+ case EWaiterType::ListTopics:
+ SendListTopicsResponse(waiter.WaiterId, ctx);
+ break;
+ case EWaiterType::DescribeAllTopics:
+ ProcessDescribeAllTopics(waiter.WaiterId, ctx);
+ break;
+ default:
+ Y_FAIL();
+ break;
+ }
+ ListTopicsWaiters.pop_front();
+ }
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE,
+ "Updated topics list with : " << CurrentTopics.size() << " topics");
+ ctx.Schedule(VersionCheckInterval, new TEvPqNewMetaCache::TEvRestartQuery(EQueryType::ECheckVersion));
+ }
+ }
+
+ void HandleQueryComplete(TEvPqNewMetaCache::TEvQueryComplete::TPtr& ev, const TActorContext& ctx) {
+ switch (ev->Get()->Type) {
+ case EQueryType::ECheckVersion:
+ return HandleCheckVersionResult(ctx);
+ case EQueryType::EGetTopics:
+ return HandleGetTopicsResult(ctx);
+ default:
+ Y_FAIL();
+ }
+ }
+
+ void HandleRestartQuery(TEvPqNewMetaCache::TEvRestartQuery::TPtr& ev, const TActorContext& ctx) {
+ Y_VERIFY(ev->Get()->Type == EQueryType::ECheckVersion);
+ RunQuery(ev->Get()->Type, ctx);
+ }
+
+ void HandleListTopics(TEvPqNewMetaCache::TEvListTopicsRequest::TPtr& ev, const TActorContext& ctx) {
+ if (!LastVersionUpdate || !EverGotTopics) {
+ ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::ListTopics});
+ } else {
+ SendListTopicsResponse(ev->Sender, ctx);
+ }
+ }
+
+ void HandleDescribeTopics(TEvPqNewMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Handle describe topics with prefix: " << ev->Get()->PathPrefix);
+
+ SendSchemeCacheRequest(ev->Get()->Topics, !ev->Get()->PathPrefix.empty(), ctx);
+ auto inserted = DescribeTopicsWaiters.insert(std::make_pair(
+ RequestId,
+ TWaiter{ev->Sender, std::move(ev->Get()->Topics), EWaiterType::DescribeCustomTopics}
+ )).second;
+ Y_VERIFY(inserted);
+ }
+
+ void HandleDescribeAllTopics(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) {
+ if (ev->Get()->PathPrefix && ev->Get()->PathPrefix != PathPrefix) {
+ auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse(ev->Get()->PathPrefix);
+ response->Success = false;
+ ctx.Send(ev->Sender, response);
+ return;
+ }
+ if (!EverGotTopics) {
+ ListTopicsWaiters.emplace_back(TWaiter{ev->Sender, {}, EWaiterType::DescribeAllTopics});
+ return;
+ }
+ return ProcessDescribeAllTopics(ev->Sender, ctx);
+ }
+
+ void ProcessDescribeAllTopics(const TActorId& waiterId, const TActorContext& ctx) {
+ if (EverGotTopics && CurrentTopics.empty()) {
+ SendDescribeAllTopicsResponse(waiterId, ctx, true);
+ return;
+ }
+ if (FullTopicsCache && !FullTopicsCacheOutdated) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Respond from cache");
+
+ return SendDescribeAllTopicsResponse(waiterId, ctx);
+ }
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Store waiter");
+ SendSchemeCacheRequest(CurrentTopics, true, ctx);
+ auto inserted = DescribeTopicsWaiters.insert(std::make_pair(
+ RequestId,
+ TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics}
+ )).second;
+ Y_VERIFY(inserted);
+ FullTopicsCacheOutdated = false;
+ FullTopicsCache = nullptr;
+ }
+
+ void HandleSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) {
+ auto* result = ev->Get()->Request.Release();
+ auto waiterIter = DescribeTopicsWaiters.find(result->Instant);
+ Y_VERIFY(!waiterIter.IsEnd());
+ auto& waiter = waiterIter->second;
+
+ if (waiter.Type == EWaiterType::DescribeAllTopics) {
for (const auto& entry : result->ResultSet) {
if (!entry.PQGroupInfo) {
continue;
@@ -342,198 +342,198 @@ private:
FullTopicsCacheOutdated = true;
}
- FullTopicsCache.Reset(result);
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Updated topics cache with " << FullTopicsCache->ResultSet.size());
- SendDescribeAllTopicsResponse(waiter.WaiterId, ctx);
- } else {
- Y_VERIFY(waiter.Type == EWaiterType::DescribeCustomTopics);
- Y_VERIFY(waiter.Topics.size() == result->ResultSet.size());
- auto *response = new TEvPqNewMetaCache::TEvDescribeTopicsResponse{
- std::move(waiter.Topics), result
- };
- ctx.Send(waiter.WaiterId, response);
- }
- DescribeTopicsWaiters.erase(waiterIter);
- }
-
-
- void SendSchemeCacheRequest(const TVector<TString>& topics, bool addDefaultPathPrefix, const TActorContext& ctx) {
+ FullTopicsCache.Reset(result);
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Updated topics cache with " << FullTopicsCache->ResultSet.size());
+ SendDescribeAllTopicsResponse(waiter.WaiterId, ctx);
+ } else {
+ Y_VERIFY(waiter.Type == EWaiterType::DescribeCustomTopics);
+ Y_VERIFY(waiter.Topics.size() == result->ResultSet.size());
+ auto *response = new TEvPqNewMetaCache::TEvDescribeTopicsResponse{
+ std::move(waiter.Topics), result
+ };
+ ctx.Send(waiter.WaiterId, response);
+ }
+ DescribeTopicsWaiters.erase(waiterIter);
+ }
+
+
+ void SendSchemeCacheRequest(const TVector<TString>& topics, bool addDefaultPathPrefix, const TActorContext& ctx) {
auto schemeCacheRequest = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(++RequestId);
- for (const auto& path : topics) {
- auto split = NKikimr::SplitPath(path);
- Y_VERIFY(!split.empty());
- NSchemeCache::TSchemeCacheNavigate::TEntry entry;
- if (addDefaultPathPrefix) {
- entry.Path.insert(entry.Path.end(), PathPrefixParts.begin(), PathPrefixParts.end());
- }
-
- entry.Path.insert(entry.Path.end(), split.begin(), split.end());
- entry.SyncVersion = true;
- entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic;
- schemeCacheRequest->ResultSet.emplace_back(std::move(entry));
- }
+ for (const auto& path : topics) {
+ auto split = NKikimr::SplitPath(path);
+ Y_VERIFY(!split.empty());
+ NSchemeCache::TSchemeCacheNavigate::TEntry entry;
+ if (addDefaultPathPrefix) {
+ entry.Path.insert(entry.Path.end(), PathPrefixParts.begin(), PathPrefixParts.end());
+ }
+
+ entry.Path.insert(entry.Path.end(), split.begin(), split.end());
+ entry.SyncVersion = true;
+ entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpTopic;
+ schemeCacheRequest->ResultSet.emplace_back(std::move(entry));
+ }
ctx.Send(SchemeCacheId, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.release()));
- }
-
- void SendListTopicsResponse(const TActorId& recipient, const TActorContext& ctx) {
- auto* response = new TEvPqNewMetaCache::TEvListTopicsResponse();
- response->Topics = CurrentTopics;
- response->TopicsVersion = CurrentTopicsVersion;
- ctx.Send(recipient, response);
- }
-
- void SendDescribeAllTopicsResponse(const TActorId& recipient, const TActorContext& ctx, bool empty = false) {
- NSchemeCache::TSchemeCacheNavigate* scResponse;
- if (empty) {
- scResponse = new NSchemeCache::TSchemeCacheNavigate();
- } else {
- scResponse = new NSchemeCache::TSchemeCacheNavigate(*FullTopicsCache);
- }
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Send describe all topics response with " << scResponse->ResultSet.size() << " topics");
- auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse(
- PathPrefix, scResponse
- );
- ctx.Send(recipient, response);
- }
-
- static NActors::TActorSystem* GetActorSystem() {
- return TActivationContext::ActorSystem();
- }
-
-public:
- void Die(const TActorContext& ctx) {
- Generation->Inc();
- TBase::Die(ctx);
- }
-
- STRICT_STFUNC(StateFunc,
- SFunc(NActors::TEvents::TEvWakeup, StartSession)
- SFunc(TEvPqNewMetaCache::TEvSessionStarted, HandleSessionStarted)
- SFunc(TEvPqNewMetaCache::TEvQueryPrepared, HandleTopicsQueryPrepared)
- HFunc(TEvPqNewMetaCache::TEvQueryComplete, HandleQueryComplete)
- HFunc(TEvPqNewMetaCache::TEvRestartQuery, HandleRestartQuery)
-
- HFunc(TEvPqNewMetaCache::TEvListTopicsRequest, HandleListTopics)
- HFunc(TEvPqNewMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics)
- HFunc(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics)
- HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse)
- )
-
-private:
- enum class EWaiterType {
- ListTopics,
- DescribeAllTopics,
- DescribeCustomTopics
- };
-
- struct TWaiter {
- TActorId WaiterId;
- TVector<TString> Topics;
- EWaiterType Type;
-
- static TWaiter AllTopicsWaiter(const TActorId& waiterId) {
- return TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics};
- }
- };
-
- struct TTopicKey {
- TString Path;
- TString Cluster;
- };
-
-
- class TClientWrapper {
- public:
+ }
+
+ void SendListTopicsResponse(const TActorId& recipient, const TActorContext& ctx) {
+ auto* response = new TEvPqNewMetaCache::TEvListTopicsResponse();
+ response->Topics = CurrentTopics;
+ response->TopicsVersion = CurrentTopicsVersion;
+ ctx.Send(recipient, response);
+ }
+
+ void SendDescribeAllTopicsResponse(const TActorId& recipient, const TActorContext& ctx, bool empty = false) {
+ NSchemeCache::TSchemeCacheNavigate* scResponse;
+ if (empty) {
+ scResponse = new NSchemeCache::TSchemeCacheNavigate();
+ } else {
+ scResponse = new NSchemeCache::TSchemeCacheNavigate(*FullTopicsCache);
+ }
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_METACACHE, "Send describe all topics response with " << scResponse->ResultSet.size() << " topics");
+ auto* response = new TEvPqNewMetaCache::TEvDescribeAllTopicsResponse(
+ PathPrefix, scResponse
+ );
+ ctx.Send(recipient, response);
+ }
+
+ static NActors::TActorSystem* GetActorSystem() {
+ return TActivationContext::ActorSystem();
+ }
+
+public:
+ void Die(const TActorContext& ctx) {
+ Generation->Inc();
+ TBase::Die(ctx);
+ }
+
+ STRICT_STFUNC(StateFunc,
+ SFunc(NActors::TEvents::TEvWakeup, StartSession)
+ SFunc(TEvPqNewMetaCache::TEvSessionStarted, HandleSessionStarted)
+ SFunc(TEvPqNewMetaCache::TEvQueryPrepared, HandleTopicsQueryPrepared)
+ HFunc(TEvPqNewMetaCache::TEvQueryComplete, HandleQueryComplete)
+ HFunc(TEvPqNewMetaCache::TEvRestartQuery, HandleRestartQuery)
+
+ HFunc(TEvPqNewMetaCache::TEvListTopicsRequest, HandleListTopics)
+ HFunc(TEvPqNewMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics)
+ HFunc(TEvPqNewMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics)
+ HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleSchemeCacheResponse)
+ )
+
+private:
+ enum class EWaiterType {
+ ListTopics,
+ DescribeAllTopics,
+ DescribeCustomTopics
+ };
+
+ struct TWaiter {
+ TActorId WaiterId;
+ TVector<TString> Topics;
+ EWaiterType Type;
+
+ static TWaiter AllTopicsWaiter(const TActorId& waiterId) {
+ return TWaiter{waiterId, {}, EWaiterType::DescribeAllTopics};
+ }
+ };
+
+ struct TTopicKey {
+ TString Path;
+ TString Cluster;
+ };
+
+
+ class TClientWrapper {
+ public:
TClientWrapper(const TClientWrapper&) = delete;
TClientWrapper& operator=(const TClientWrapper&) = delete;
TClientWrapper(TClientWrapper&&) = default;
TClientWrapper& operator=(TClientWrapper&&) = default;
- TClientWrapper(ui64 driverPort)
- : DriverPort(driverPort)
- {}
-
- TClientWrapper(NYdb::TDriver* driver)
- : Driver(driver)
- {}
-
- void Reset() {
- if (DriverPort.Defined()) { // Own driver
- if (DriverHolder == nullptr) {
- TString endpoint = TStringBuilder() << "localhost:" << *DriverPort;
- auto driverConfig = NYdb::TDriverConfig().SetEndpoint(endpoint);
- DriverHolder.Reset(new NYdb::TDriver(driverConfig));
- Driver = DriverHolder.Get();
+ TClientWrapper(ui64 driverPort)
+ : DriverPort(driverPort)
+ {}
+
+ TClientWrapper(NYdb::TDriver* driver)
+ : Driver(driver)
+ {}
+
+ void Reset() {
+ if (DriverPort.Defined()) { // Own driver
+ if (DriverHolder == nullptr) {
+ TString endpoint = TStringBuilder() << "localhost:" << *DriverPort;
+ auto driverConfig = NYdb::TDriverConfig().SetEndpoint(endpoint);
+ DriverHolder.Reset(new NYdb::TDriver(driverConfig));
+ Driver = DriverHolder.Get();
TableClient.reset(new NYdb::NTable::TTableClient(*Driver));
- }
- } else if (Driver != nullptr) {
+ }
+ } else if (Driver != nullptr) {
TableClient.reset(new NYdb::NTable::TTableClient(*Driver));
- }
- }
+ }
+ }
- NYdb::NTable::TTableClient* GetClient() {
- Y_VERIFY(TableClient);
+ NYdb::NTable::TTableClient* GetClient() {
+ Y_VERIFY(TableClient);
return TableClient.get();
- }
-
- void Stop() {
- if (DriverHolder != nullptr) {
- TableClient->Stop();
- DriverHolder->Stop();
- } else if (Driver != nullptr) {
- TableClient->Stop();
- }
- }
-
- private:
- THolder<NYdb::TDriver> DriverHolder;
- NYdb::TDriver* Driver = nullptr;
- TMaybe<ui64> DriverPort;
+ }
+
+ void Stop() {
+ if (DriverHolder != nullptr) {
+ TableClient->Stop();
+ DriverHolder->Stop();
+ } else if (Driver != nullptr) {
+ TableClient->Stop();
+ }
+ }
+
+ private:
+ THolder<NYdb::TDriver> DriverHolder;
+ NYdb::TDriver* Driver = nullptr;
+ TMaybe<ui64> DriverPort;
std::unique_ptr<NYdb::NTable::TTableClient> TableClient;
};
-
- NMonitoring::TDynamicCounterPtr Counters;
- NActors::TActorSystem* ActorSystem;
- TString VersionQuery;
- TString TopicsQuery;
-
+
+ NMonitoring::TDynamicCounterPtr Counters;
+ NActors::TActorSystem* ActorSystem;
+ TString VersionQuery;
+ TString TopicsQuery;
+
std::unique_ptr<TClientWrapper> ClientWrapper;
- TAsyncCreateSessionResult SessionFuture;
- TMaybe<TSession> YdbSession;
- TAsyncPrepareQueryResult TopicsQueryFuture;
- TMaybe<TDataQuery> PreparedTopicsQuery;
- TAsyncDataQueryResult AsyncQueryResult;
- ui64 CurrentTopicsVersion = 0;
- ui64 NewTopicsVersion = 0;
- TTopicKey LastTopicKey = TTopicKey{};
- TVector<TString> NewTopics;
- TVector<TString> CurrentTopics;
- bool EverGotTopics = false;
- TDuration QueryRetryInterval = TDuration::Seconds(2);
- TDuration VersionCheckInterval = TDuration::Seconds(1);
- TInstant LastVersionUpdate = TInstant::Zero();
-
- TDeque<TWaiter> ListTopicsWaiters;
- THashMap<ui64, TWaiter> DescribeTopicsWaiters;
- ui64 RequestId = 1;
-
- THolder<NSchemeCache::TSchemeCacheNavigate> FullTopicsCache;
- bool FullTopicsCacheOutdated = false;
- NActors::TActorId SchemeCacheId;
- TString PathPrefix;
- TVector<TString> PathPrefixParts;
+ TAsyncCreateSessionResult SessionFuture;
+ TMaybe<TSession> YdbSession;
+ TAsyncPrepareQueryResult TopicsQueryFuture;
+ TMaybe<TDataQuery> PreparedTopicsQuery;
+ TAsyncDataQueryResult AsyncQueryResult;
+ ui64 CurrentTopicsVersion = 0;
+ ui64 NewTopicsVersion = 0;
+ TTopicKey LastTopicKey = TTopicKey{};
+ TVector<TString> NewTopics;
+ TVector<TString> CurrentTopics;
+ bool EverGotTopics = false;
+ TDuration QueryRetryInterval = TDuration::Seconds(2);
+ TDuration VersionCheckInterval = TDuration::Seconds(1);
+ TInstant LastVersionUpdate = TInstant::Zero();
+
+ TDeque<TWaiter> ListTopicsWaiters;
+ THashMap<ui64, TWaiter> DescribeTopicsWaiters;
+ ui64 RequestId = 1;
+
+ THolder<NSchemeCache::TSchemeCacheNavigate> FullTopicsCache;
+ bool FullTopicsCacheOutdated = false;
+ NActors::TActorId SchemeCacheId;
+ TString PathPrefix;
+ TVector<TString> PathPrefixParts;
std::shared_ptr<TAtomicCounter> Generation;
- bool SkipVersionCheck = false;
-};
-
-IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) {
- return new TPersQueueMetaCacheActor(counters, versionCheckInterval);
-}
-
-IActor* CreatePQMetaCache(ui64 grpcPort, const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) {
- return new TPersQueueMetaCacheActor(grpcPort, counters, versionCheckInterval);
-}
-
-} // namespace NPqMetaCacheV2
-
-} // namespace NKikimr::NMsgBusProxy
+ bool SkipVersionCheck = false;
+};
+
+IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) {
+ return new TPersQueueMetaCacheActor(counters, versionCheckInterval);
+}
+
+IActor* CreatePQMetaCache(ui64 grpcPort, const NMonitoring::TDynamicCounterPtr& counters, const TDuration& versionCheckInterval) {
+ return new TPersQueueMetaCacheActor(grpcPort, counters, versionCheckInterval);
+}
+
+} // namespace NPqMetaCacheV2
+
+} // namespace NKikimr::NMsgBusProxy
diff --git a/ydb/core/client/server/msgbus_server_pq_metacache.h b/ydb/core/client/server/msgbus_server_pq_metacache.h
index 8a1822dab3..6c12b76b04 100644
--- a/ydb/core/client/server/msgbus_server_pq_metacache.h
+++ b/ydb/core/client/server/msgbus_server_pq_metacache.h
@@ -3,140 +3,140 @@
#include <library/cpp/actors/core/events.h>
#include <library/cpp/actors/core/event_local.h>
#include <ydb/core/tx/schemeshard/schemeshard.h>
-
+
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/public/api/protos/draft/persqueue_error_codes.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <util/generic/string.h>
#include <util/generic/vector.h>
-namespace NKikimr::NMsgBusProxy {
+namespace NKikimr::NMsgBusProxy {
static const ui32 PQ_METACACHE_TIMEOUT_SECONDS = 120;
-static const ui32 PQ_METACACHE_REFRESH_INTERVAL_SECONDS = 10;
+static const ui32 PQ_METACACHE_REFRESH_INTERVAL_SECONDS = 10;
-inline TActorId CreatePersQueueMetaCacheV2Id() {
+inline TActorId CreatePersQueueMetaCacheV2Id() {
return TActorId(0, "PQMetaCache");
}
-namespace NPqMetaCacheV2 {
-
-enum class EQueryType {
- ECheckVersion,
- EGetTopics,
-};
-
-struct TTopicMetaRequest {
- TString Path;
- THolder<NSchemeCache::TSchemeCacheNavigate> Response;
- bool Success = false;
-};
-
-using TMetaCacheRequest = TVector<TTopicMetaRequest>;
-
-struct TEvPqNewMetaCache {
- enum EEv {
- EvWakeup = EventSpaceBegin(TKikimrEvents::ES_PQ_META_CACHE),
- EvSessionStarted,
- EvQueryPrepared,
- EvQueryComplete,
- EvRestartQuery,
- EvListTopicsRequest,
- EvListTopicsResponse,
- EvDescribeTopicsRequest,
- EvDescribeTopicsResponse,
- EvDescribeAllTopicsRequest,
- EvDescribeAllTopicsResponse,
- EvEnd
- };
-
- static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE),
- "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE)");
-
- struct TEvSessionStarted : public TEventLocal<TEvSessionStarted, EvSessionStarted> {
- };
-
- struct TEvQueryPrepared : public TEventLocal<TEvQueryPrepared, EvQueryPrepared> {
- };
-
- struct TEvQueryComplete : public TEventLocal<TEvQueryComplete, EvQueryComplete> {
- EQueryType Type;
-
- explicit TEvQueryComplete(EQueryType type)
- : Type(type) {}
- };
-
- struct TEvRestartQuery : public TEventLocal<TEvRestartQuery, EvRestartQuery> {
- EQueryType Type;
-
- explicit TEvRestartQuery(EQueryType type)
- : Type(type) {}
- };
-
- struct TEvListTopicsRequest : public TEventLocal<TEvListTopicsRequest, EvListTopicsRequest> {
- };
-
- struct TEvListTopicsResponse : public TEventLocal<TEvListTopicsResponse, EvListTopicsResponse> {
- TVector<TString> Topics;
- ui64 TopicsVersion;
- };
-
- struct TEvDescribeTopicsRequest : public TEventLocal<TEvDescribeTopicsRequest, EvDescribeTopicsRequest> {
- TString PathPrefix;
- TVector<TString> Topics;
-
- TEvDescribeTopicsRequest() = default;
-
- explicit TEvDescribeTopicsRequest(const TVector<TString>& topics)
- : Topics(topics)
- {}
-
- TEvDescribeTopicsRequest(const TVector<TString>& topics, const TString& pathPrefix)
- : PathPrefix(pathPrefix)
- , Topics(topics)
- {}
- };
-
- struct TEvDescribeTopicsResponse : public TEventLocal<TEvDescribeTopicsResponse, EvDescribeTopicsResponse> {
- TVector<TString> TopicsRequested;
- THolder<NSchemeCache::TSchemeCacheNavigate> Result;
- explicit TEvDescribeTopicsResponse(const TVector<TString>& topics,
- NSchemeCache::TSchemeCacheNavigate* result)
-
- : TopicsRequested(topics)
- , Result(result)
- {}
- };
-
- struct TEvDescribeAllTopicsRequest : public TEventLocal<TEvDescribeAllTopicsRequest, EvDescribeAllTopicsRequest> {
- TString PathPrefix;
- TEvDescribeAllTopicsRequest() = default;
- explicit TEvDescribeAllTopicsRequest(const TString& pathPrefix)
- : PathPrefix(pathPrefix)
- {}
- };
-
- struct TEvDescribeAllTopicsResponse : public TEventLocal<TEvDescribeAllTopicsResponse, EvDescribeAllTopicsResponse> {
- bool Success = true;
- TString Path;
- THolder<NSchemeCache::TSchemeCacheNavigate> Result;
- explicit TEvDescribeAllTopicsResponse(const TString& path)
- : Path(path)
- {}
- TEvDescribeAllTopicsResponse(const TString& path, NSchemeCache::TSchemeCacheNavigate* result)
- : Path(path)
- , Result(result)
- {}
- };
-};
-IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters,
- const TDuration& versionCheckInterval);
-
-IActor* CreatePQMetaCache(ui64 grpcPort,
- const NMonitoring::TDynamicCounterPtr& counters,
- const TDuration& versionCheckInterval = TDuration::Seconds(1));
-
-} // namespace NPqMetaCacheV2
-
-} //namespace NKikimr::NMsgBusProxy
+namespace NPqMetaCacheV2 {
+
+enum class EQueryType {
+ ECheckVersion,
+ EGetTopics,
+};
+
+struct TTopicMetaRequest {
+ TString Path;
+ THolder<NSchemeCache::TSchemeCacheNavigate> Response;
+ bool Success = false;
+};
+
+using TMetaCacheRequest = TVector<TTopicMetaRequest>;
+
+struct TEvPqNewMetaCache {
+ enum EEv {
+ EvWakeup = EventSpaceBegin(TKikimrEvents::ES_PQ_META_CACHE),
+ EvSessionStarted,
+ EvQueryPrepared,
+ EvQueryComplete,
+ EvRestartQuery,
+ EvListTopicsRequest,
+ EvListTopicsResponse,
+ EvDescribeTopicsRequest,
+ EvDescribeTopicsResponse,
+ EvDescribeAllTopicsRequest,
+ EvDescribeAllTopicsResponse,
+ EvEnd
+ };
+
+ static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE),
+ "expect EvEnd < EventSpaceEnd(TKikimrEvents::ES_PQ_META_CACHE)");
+
+ struct TEvSessionStarted : public TEventLocal<TEvSessionStarted, EvSessionStarted> {
+ };
+
+ struct TEvQueryPrepared : public TEventLocal<TEvQueryPrepared, EvQueryPrepared> {
+ };
+
+ struct TEvQueryComplete : public TEventLocal<TEvQueryComplete, EvQueryComplete> {
+ EQueryType Type;
+
+ explicit TEvQueryComplete(EQueryType type)
+ : Type(type) {}
+ };
+
+ struct TEvRestartQuery : public TEventLocal<TEvRestartQuery, EvRestartQuery> {
+ EQueryType Type;
+
+ explicit TEvRestartQuery(EQueryType type)
+ : Type(type) {}
+ };
+
+ struct TEvListTopicsRequest : public TEventLocal<TEvListTopicsRequest, EvListTopicsRequest> {
+ };
+
+ struct TEvListTopicsResponse : public TEventLocal<TEvListTopicsResponse, EvListTopicsResponse> {
+ TVector<TString> Topics;
+ ui64 TopicsVersion;
+ };
+
+ struct TEvDescribeTopicsRequest : public TEventLocal<TEvDescribeTopicsRequest, EvDescribeTopicsRequest> {
+ TString PathPrefix;
+ TVector<TString> Topics;
+
+ TEvDescribeTopicsRequest() = default;
+
+ explicit TEvDescribeTopicsRequest(const TVector<TString>& topics)
+ : Topics(topics)
+ {}
+
+ TEvDescribeTopicsRequest(const TVector<TString>& topics, const TString& pathPrefix)
+ : PathPrefix(pathPrefix)
+ , Topics(topics)
+ {}
+ };
+
+ struct TEvDescribeTopicsResponse : public TEventLocal<TEvDescribeTopicsResponse, EvDescribeTopicsResponse> {
+ TVector<TString> TopicsRequested;
+ THolder<NSchemeCache::TSchemeCacheNavigate> Result;
+ explicit TEvDescribeTopicsResponse(const TVector<TString>& topics,
+ NSchemeCache::TSchemeCacheNavigate* result)
+
+ : TopicsRequested(topics)
+ , Result(result)
+ {}
+ };
+
+ struct TEvDescribeAllTopicsRequest : public TEventLocal<TEvDescribeAllTopicsRequest, EvDescribeAllTopicsRequest> {
+ TString PathPrefix;
+ TEvDescribeAllTopicsRequest() = default;
+ explicit TEvDescribeAllTopicsRequest(const TString& pathPrefix)
+ : PathPrefix(pathPrefix)
+ {}
+ };
+
+ struct TEvDescribeAllTopicsResponse : public TEventLocal<TEvDescribeAllTopicsResponse, EvDescribeAllTopicsResponse> {
+ bool Success = true;
+ TString Path;
+ THolder<NSchemeCache::TSchemeCacheNavigate> Result;
+ explicit TEvDescribeAllTopicsResponse(const TString& path)
+ : Path(path)
+ {}
+ TEvDescribeAllTopicsResponse(const TString& path, NSchemeCache::TSchemeCacheNavigate* result)
+ : Path(path)
+ , Result(result)
+ {}
+ };
+};
+IActor* CreatePQMetaCache(const NMonitoring::TDynamicCounterPtr& counters,
+ const TDuration& versionCheckInterval);
+
+IActor* CreatePQMetaCache(ui64 grpcPort,
+ const NMonitoring::TDynamicCounterPtr& counters,
+ const TDuration& versionCheckInterval = TDuration::Seconds(1));
+
+} // namespace NPqMetaCacheV2
+
+} //namespace NKikimr::NMsgBusProxy
diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp
index 5aac260ffe..32f37251cc 100644
--- a/ydb/core/client/server/msgbus_server_pq_metarequest.cpp
+++ b/ydb/core/client/server/msgbus_server_pq_metarequest.cpp
@@ -3,16 +3,16 @@
namespace NKikimr {
namespace NMsgBusProxy {
-using namespace NSchemeCache;
+using namespace NSchemeCache;
template <class TTopicResult>
-void SetErrorCode(
- TTopicResult* topicResult,
- const TSchemeCacheNavigate::TEntry& topicEntry
-) {
- NPersQueue::NErrorCode::EErrorCode code = NPersQueue::NErrorCode::OK;
- if (topicEntry.Status != TSchemeCacheNavigate::EStatus::Ok || !topicEntry.PQGroupInfo) {
- code = NPersQueue::NErrorCode::UNKNOWN_TOPIC;
+void SetErrorCode(
+ TTopicResult* topicResult,
+ const TSchemeCacheNavigate::TEntry& topicEntry
+) {
+ NPersQueue::NErrorCode::EErrorCode code = NPersQueue::NErrorCode::OK;
+ if (topicEntry.Status != TSchemeCacheNavigate::EStatus::Ok || !topicEntry.PQGroupInfo) {
+ code = NPersQueue::NErrorCode::UNKNOWN_TOPIC;
}
topicResult->SetErrorCode(code);
if (code == NPersQueue::NErrorCode::UNKNOWN_TOPIC) {
@@ -38,24 +38,24 @@ TPersQueueGetTopicMetadataProcessor::TPersQueueGetTopicMetadataProcessor(const N
}
}
-THolder<IActor> TPersQueueGetTopicMetadataProcessor::CreateTopicSubactor(
- const TSchemeEntry& topicEntry, const TString& name
-) {
- return MakeHolder<TPersQueueGetTopicMetadataTopicWorker>(SelfId(), topicEntry, name);
+THolder<IActor> TPersQueueGetTopicMetadataProcessor::CreateTopicSubactor(
+ const TSchemeEntry& topicEntry, const TString& name
+) {
+ return MakeHolder<TPersQueueGetTopicMetadataTopicWorker>(SelfId(), topicEntry, name);
}
-TPersQueueGetTopicMetadataTopicWorker::TPersQueueGetTopicMetadataTopicWorker(
- const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name
-)
- : TReplierToParent<TTopicInfoBasedActor>(parent, topicEntry, name)
+TPersQueueGetTopicMetadataTopicWorker::TPersQueueGetTopicMetadataTopicWorker(
+ const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name
+)
+ : TReplierToParent<TTopicInfoBasedActor>(parent, topicEntry, name)
{
SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR);
}
-void TPersQueueGetTopicMetadataTopicWorker::BootstrapImpl(const TActorContext& ctx) {
- auto processingResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
- Answer(ctx, processingResult.Status, processingResult.ErrorCode, processingResult.Reason);
+void TPersQueueGetTopicMetadataTopicWorker::BootstrapImpl(const TActorContext& ctx) {
+ auto processingResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
+ Answer(ctx, processingResult.Status, processingResult.ErrorCode, processingResult.Reason);
}
void TPersQueueGetTopicMetadataTopicWorker::Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) {
@@ -66,13 +66,13 @@ void TPersQueueGetTopicMetadataTopicWorker::Answer(const TActorContext& ctx, ERe
response.SetErrorReason(errorReason);
if (code == NPersQueue::NErrorCode::OK) {
auto* topicInfo = response.MutableMetaResponse()->MutableCmdGetTopicMetadataResult()->AddTopicInfo();
- SetErrorCode(topicInfo, SchemeEntry);
- if (SchemeEntry.PQGroupInfo != nullptr) {
- const auto& desc = SchemeEntry.PQGroupInfo->Description;
- topicInfo->SetTopic(desc.GetName());
- topicInfo->MutableConfig()->CopyFrom(desc.GetPQTabletConfig());
- topicInfo->MutableConfig()->SetVersion(desc.GetAlterVersion());
- topicInfo->SetNumPartitions(desc.PartitionsSize());
+ SetErrorCode(topicInfo, SchemeEntry);
+ if (SchemeEntry.PQGroupInfo != nullptr) {
+ const auto& desc = SchemeEntry.PQGroupInfo->Description;
+ topicInfo->SetTopic(desc.GetName());
+ topicInfo->MutableConfig()->CopyFrom(desc.GetPQTabletConfig());
+ topicInfo->MutableConfig()->SetVersion(desc.GetAlterVersion());
+ topicInfo->SetNumPartitions(desc.PartitionsSize());
}
}
SendReplyAndDie(std::move(response), ctx);
@@ -83,41 +83,41 @@ void TPersQueueGetTopicMetadataTopicWorker::Answer(const TActorContext& ctx, ERe
// GetPartitionOffsets command
//
-TPersQueueGetPartitionOffsetsProcessor::TPersQueueGetPartitionOffsetsProcessor(
- const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId
-)
- : TPersQueueBaseRequestProcessor(request, metaCacheId, false)
+TPersQueueGetPartitionOffsetsProcessor::TPersQueueGetPartitionOffsetsProcessor(
+ const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId
+)
+ : TPersQueueBaseRequestProcessor(request, metaCacheId, false)
{
GetTopicsListOrThrow(RequestProto->GetMetaRequest().GetCmdGetPartitionOffsets().GetTopicRequest(), PartitionsToRequest);
}
-THolder<IActor> TPersQueueGetPartitionOffsetsProcessor::CreateTopicSubactor(
- const TSchemeEntry& topicEntry, const TString& name
-) {
- return MakeHolder<TPersQueueGetPartitionOffsetsTopicWorker>(
- SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto
- );
+THolder<IActor> TPersQueueGetPartitionOffsetsProcessor::CreateTopicSubactor(
+ const TSchemeEntry& topicEntry, const TString& name
+) {
+ return MakeHolder<TPersQueueGetPartitionOffsetsTopicWorker>(
+ SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto
+ );
}
-TPersQueueGetPartitionOffsetsTopicWorker::TPersQueueGetPartitionOffsetsTopicWorker(
- const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name,
+TPersQueueGetPartitionOffsetsTopicWorker::TPersQueueGetPartitionOffsetsTopicWorker(
+ const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name,
const std::shared_ptr<THashSet<ui64>>& partitionsToRequest,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto
-)
- : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name)
+)
+ : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name)
, PartitionsToRequest(partitionsToRequest)
, RequestProto(requestProto)
{
SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR);
}
-void TPersQueueGetPartitionOffsetsTopicWorker::BootstrapImpl(const TActorContext &ctx) {
+void TPersQueueGetPartitionOffsetsTopicWorker::BootstrapImpl(const TActorContext &ctx) {
size_t partitionsAsked = 0;
THashSet<ui64> parts;
- if (SchemeEntry.PQGroupInfo) {
- const auto& pqDescr = SchemeEntry.PQGroupInfo->Description;
- for (const auto& partition : pqDescr.GetPartitions()) {
- const ui32 partIndex = partition.GetPartitionId();
+ if (SchemeEntry.PQGroupInfo) {
+ const auto& pqDescr = SchemeEntry.PQGroupInfo->Description;
+ for (const auto& partition : pqDescr.GetPartitions()) {
+ const ui32 partIndex = partition.GetPartitionId();
const ui64 tabletId = partition.GetTabletId();
if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) {
continue;
@@ -137,22 +137,22 @@ void TPersQueueGetPartitionOffsetsTopicWorker::BootstrapImpl(const TActorContext
}
if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) {
SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ96");
- return;
+ TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ96");
+ return;
}
if (!PartitionsToRequest.get() || PartitionsToRequest->empty()) {
PartitionsToRequest.reset(new THashSet<ui64>());
PartitionsToRequest->swap(parts);
}
- if(WaitAllPipeEvents(ctx)) {
- return;
- }
+ if(WaitAllPipeEvents(ctx)) {
+ return;
+ }
}
bool TPersQueueGetPartitionOffsetsTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) {
- auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
- Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason);
-
+ auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
+ Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason);
+
return true;
}
@@ -164,8 +164,8 @@ void TPersQueueGetPartitionOffsetsTopicWorker::Answer(const TActorContext& ctx,
response.SetErrorReason(errorReason);
if (code == NPersQueue::NErrorCode::OK) {
auto& topicResult = *response.MutableMetaResponse()->MutableCmdGetPartitionOffsetsResult()->AddTopicResult();
- topicResult.SetTopic(Name);
- SetErrorCode(&topicResult, SchemeEntry);
+ topicResult.SetTopic(Name);
+ SetErrorCode(&topicResult, SchemeEntry);
THashSet<ui64> partitionsInserted;
for (auto& ans : PipeAnswers) {
if (ans.second.Get() != nullptr) {
@@ -204,35 +204,35 @@ TPersQueueGetPartitionStatusProcessor::TPersQueueGetPartitionStatusProcessor(con
GetTopicsListOrThrow(RequestProto->GetMetaRequest().GetCmdGetPartitionStatus().GetTopicRequest(), PartitionsToRequest);
}
-THolder<IActor> TPersQueueGetPartitionStatusProcessor::CreateTopicSubactor(
- const TSchemeEntry& topicEntry, const TString& name
-) {
- return MakeHolder<TPersQueueGetPartitionStatusTopicWorker>(
- SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto
- );
+THolder<IActor> TPersQueueGetPartitionStatusProcessor::CreateTopicSubactor(
+ const TSchemeEntry& topicEntry, const TString& name
+) {
+ return MakeHolder<TPersQueueGetPartitionStatusTopicWorker>(
+ SelfId(), topicEntry, name, PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto
+ );
}
-TPersQueueGetPartitionStatusTopicWorker::TPersQueueGetPartitionStatusTopicWorker(
- const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name,
+TPersQueueGetPartitionStatusTopicWorker::TPersQueueGetPartitionStatusTopicWorker(
+ const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name,
const std::shared_ptr<THashSet<ui64>>& partitionsToRequest,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto
-)
- : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>>(parent, topicEntry, name)
+)
+ : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>>(parent, topicEntry, name)
, PartitionsToRequest(partitionsToRequest)
, RequestProto(requestProto)
{
SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR);
}
-void TPersQueueGetPartitionStatusTopicWorker::BootstrapImpl(const TActorContext &ctx) {
+void TPersQueueGetPartitionStatusTopicWorker::BootstrapImpl(const TActorContext &ctx) {
size_t partitionsAsked = 0;
THashSet<ui64> parts;
- if (!ProcessingResult.IsFatal) {
- const auto& pqDescr = SchemeEntry.PQGroupInfo->Description;
- for (const auto& partition : pqDescr.GetPartitions()) {
- const ui32 partIndex = partition.GetPartitionId();
+ if (!ProcessingResult.IsFatal) {
+ const auto& pqDescr = SchemeEntry.PQGroupInfo->Description;
+ for (const auto& partition : pqDescr.GetPartitions()) {
+ const ui32 partIndex = partition.GetPartitionId();
const ui64 tabletId = partition.GetTabletId();
- if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) {
+ if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) {
continue;
}
parts.insert(partIndex);
@@ -245,34 +245,34 @@ void TPersQueueGetPartitionStatusTopicWorker::BootstrapImpl(const TActorContext
ev->Record.SetClientId(RequestProto->GetMetaRequest().GetCmdGetPartitionStatus().GetClientId());
CreatePipeAndSend(tabletId, ctx, std::move(ev));
}
- } else {
- SendErrorReplyAndDie(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
- return;
+ } else {
+ SendErrorReplyAndDie(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
+ return;
}
- if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) {
+ if (PartitionsToRequest != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) {
SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ97");
- return;
+ TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ97");
+ return;
}
if (!PartitionsToRequest.get() || PartitionsToRequest->empty()) {
PartitionsToRequest.reset(new THashSet<ui64>());
PartitionsToRequest->swap(parts);
}
- if (WaitAllPipeEvents(ctx))
- return;
+ if (WaitAllPipeEvents(ctx))
+ return;
}
bool TPersQueueGetPartitionStatusTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) {
- auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
- Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason);
+ auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
+ Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason);
return true;
}
-void TPersQueueGetPartitionStatusTopicWorker::Answer(
- const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code,
- const TString& errorReason
-) {
+void TPersQueueGetPartitionStatusTopicWorker::Answer(
+ const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code,
+ const TString& errorReason
+) {
NKikimrClient::TResponse response;
response.SetStatus(status);
response.SetErrorCode(code);
@@ -280,8 +280,8 @@ void TPersQueueGetPartitionStatusTopicWorker::Answer(
response.SetErrorReason(errorReason);
if (code == NPersQueue::NErrorCode::OK) {
auto& topicResult = *response.MutableMetaResponse()->MutableCmdGetPartitionStatusResult()->AddTopicResult();
- topicResult.SetTopic(Name);
- SetErrorCode(&topicResult, SchemeEntry);
+ topicResult.SetTopic(Name);
+ SetErrorCode(&topicResult, SchemeEntry);
THashSet<ui64> partitionsInserted;
for (auto& ans : PipeAnswers) {
if (ans.second.Get() != nullptr) {
@@ -315,32 +315,32 @@ void TPersQueueGetPartitionStatusTopicWorker::Answer(
// GetPartitionLocations command
//
-TPersQueueGetPartitionLocationsProcessor::TPersQueueGetPartitionLocationsProcessor(
- const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache
-)
+TPersQueueGetPartitionLocationsProcessor::TPersQueueGetPartitionLocationsProcessor(
+ const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache
+)
: TPersQueueBaseRequestProcessor(request, schemeCache, true)
{
GetTopicsListOrThrow(RequestProto->GetMetaRequest().GetCmdGetPartitionLocations().GetTopicRequest(), PartitionsToRequest);
}
-THolder<IActor> TPersQueueGetPartitionLocationsProcessor::CreateTopicSubactor(
- const TSchemeEntry& topicEntry, const TString& name
-) {
+THolder<IActor> TPersQueueGetPartitionLocationsProcessor::CreateTopicSubactor(
+ const TSchemeEntry& topicEntry, const TString& name
+) {
Y_VERIFY(NodesInfo.get() != nullptr);
- return MakeHolder<TPersQueueGetPartitionLocationsTopicWorker>(
- SelfId(), topicEntry, name,
- PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto, NodesInfo
- );
+ return MakeHolder<TPersQueueGetPartitionLocationsTopicWorker>(
+ SelfId(), topicEntry, name,
+ PartitionsToRequest[topicEntry.PQGroupInfo->Description.GetName()], RequestProto, NodesInfo
+ );
}
-TPersQueueGetPartitionLocationsTopicWorker::TPersQueueGetPartitionLocationsTopicWorker(
- const TActorId& parent,
- const TSchemeEntry& topicEntry, const TString& name,
+TPersQueueGetPartitionLocationsTopicWorker::TPersQueueGetPartitionLocationsTopicWorker(
+ const TActorId& parent,
+ const TSchemeEntry& topicEntry, const TString& name,
const std::shared_ptr<THashSet<ui64>>& partitionsToRequest,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto,
std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo
-)
- : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>>(parent, topicEntry, name)
+)
+ : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>>(parent, topicEntry, name)
, PartitionsToRequest(partitionsToRequest)
, RequestProto(requestProto)
, NodesInfo(nodesInfo)
@@ -348,13 +348,13 @@ TPersQueueGetPartitionLocationsTopicWorker::TPersQueueGetPartitionLocationsTopic
SetActivityType(NKikimrServices::TActivity::PQ_META_REQUEST_PROCESSOR);
}
-void TPersQueueGetPartitionLocationsTopicWorker::BootstrapImpl(const TActorContext& ctx) {
+void TPersQueueGetPartitionLocationsTopicWorker::BootstrapImpl(const TActorContext& ctx) {
size_t partitionsAsked = 0;
THashSet<ui64> parts;
- if (SchemeEntry.PQGroupInfo) {
- const auto& pqDescr = SchemeEntry.PQGroupInfo->Description;
- for (const auto& partition : pqDescr.GetPartitions()) {
- const ui32 partIndex = partition.GetPartitionId();
+ if (SchemeEntry.PQGroupInfo) {
+ const auto& pqDescr = SchemeEntry.PQGroupInfo->Description;
+ for (const auto& partition : pqDescr.GetPartitions()) {
+ const ui32 partIndex = partition.GetPartitionId();
const ui64 tabletId = partition.GetTabletId();
if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && !PartitionsToRequest->contains(partIndex)) {
continue;
@@ -370,28 +370,28 @@ void TPersQueueGetPartitionLocationsTopicWorker::BootstrapImpl(const TActorConte
}
if (PartitionsToRequest.get() != nullptr && !PartitionsToRequest->empty() && PartitionsToRequest->size() != partitionsAsked) {
SendErrorReplyAndDie(ctx, MSTATUS_ERROR, NPersQueue::NErrorCode::UNKNOWN_TOPIC,
- TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ98");
- return;
+ TStringBuilder() << "no one of requested partitions in topic '" << Name << "', Marker# PQ98");
+ return;
}
if (!PartitionsToRequest.get() || PartitionsToRequest->empty()) {
PartitionsToRequest.reset(new THashSet<ui64>());
PartitionsToRequest->swap(parts);
}
- if(WaitAllConnections(ctx))
- return;
+ if(WaitAllConnections(ctx))
+ return;
}
bool TPersQueueGetPartitionLocationsTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) {
- auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
- Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason);
+ auto processResult = ProcessMetaCacheSingleTopicsResponse(SchemeEntry);
+ Answer(ctx, processResult.Status, processResult.ErrorCode, processResult.Reason);
return true;
}
-void TPersQueueGetPartitionLocationsTopicWorker::Answer(
- const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code,
- const TString& errorReason
-) {
+void TPersQueueGetPartitionLocationsTopicWorker::Answer(
+ const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code,
+ const TString& errorReason
+) {
NKikimrClient::TResponse response;
response.SetStatus(status);
response.SetErrorCode(code);
@@ -400,8 +400,8 @@ void TPersQueueGetPartitionLocationsTopicWorker::Answer(
if (code == NPersQueue::NErrorCode::OK) {
auto& topicResult = *response.MutableMetaResponse()->MutableCmdGetPartitionLocationsResult()->AddTopicResult();
- topicResult.SetTopic(Name);
- SetErrorCode(&topicResult, SchemeEntry);
+ topicResult.SetTopic(Name);
+ SetErrorCode(&topicResult, SchemeEntry);
for (const auto& partitionToTablet : PartitionToTablet) {
const ui32 partition = partitionToTablet.first;
const ui64 tabletId = partitionToTablet.second;
@@ -438,11 +438,11 @@ void TPersQueueGetPartitionLocationsTopicWorker::Answer(
// GetReadSessionsInfo command
//
-TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor(
+TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor(
const NKikimrClient::TPersQueueRequest& request,
const TActorId& schemeCache,
std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> pqReadSessionsInfoWorkerFactory
-)
+)
: TPersQueueBaseRequestProcessor(request, schemeCache, true)
, PQReadSessionsInfoWorkerFactory(pqReadSessionsInfoWorkerFactory)
{
@@ -458,20 +458,20 @@ TPersQueueGetReadSessionsInfoProcessor::TPersQueueGetReadSessionsInfoProcessor(
}
}
-THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateTopicSubactor(
- const TSchemeEntry& topicEntry, const TString& name
-) {
+THolder<IActor> TPersQueueGetReadSessionsInfoProcessor::CreateTopicSubactor(
+ const TSchemeEntry& topicEntry, const TString& name
+) {
Y_VERIFY(NodesInfo.get() != nullptr);
- return MakeHolder<TPersQueueGetReadSessionsInfoTopicWorker>(
- SelfId(), topicEntry, name, RequestProto, NodesInfo);
+ return MakeHolder<TPersQueueGetReadSessionsInfoTopicWorker>(
+ SelfId(), topicEntry, name, RequestProto, NodesInfo);
}
-TPersQueueGetReadSessionsInfoTopicWorker::TPersQueueGetReadSessionsInfoTopicWorker(
- const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name,
+TPersQueueGetReadSessionsInfoTopicWorker::TPersQueueGetReadSessionsInfoTopicWorker(
+ const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto,
std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo
-)
- : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name)
+)
+ : TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>>(parent, topicEntry, name)
, RequestProto(requestProto)
, NodesInfo(nodesInfo)
{
@@ -487,20 +487,20 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Die(const TActorContext& ctx) {
void TPersQueueGetReadSessionsInfoTopicWorker::SendReadSessionsInfoToBalancer(const TActorContext& ctx) {
NTabletPipe::TClientConfig clientConfig;
- BalancerPipe = ctx.RegisterWithSameMailbox(
- NTabletPipe::CreateClient(ctx.SelfID, SchemeEntry.PQGroupInfo->Description.GetBalancerTabletID(), clientConfig)
- );
+ BalancerPipe = ctx.RegisterWithSameMailbox(
+ NTabletPipe::CreateClient(ctx.SelfID, SchemeEntry.PQGroupInfo->Description.GetBalancerTabletID(), clientConfig)
+ );
THolder<TEvPersQueue::TEvGetReadSessionsInfo> ev(new TEvPersQueue::TEvGetReadSessionsInfo());
ev->Record.SetClientId(RequestProto->GetMetaRequest().GetCmdGetReadSessionsInfo().GetClientId());
NTabletPipe::SendData(ctx, BalancerPipe, ev.Release());
}
-void TPersQueueGetReadSessionsInfoTopicWorker::BootstrapImpl(const TActorContext &ctx) {
- if (!ProcessingResult.IsFatal) {
- SendReadSessionsInfoToBalancer(ctx);
- for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) {
- const ui32 partitionIndex = partition.GetPartitionId();
+void TPersQueueGetReadSessionsInfoTopicWorker::BootstrapImpl(const TActorContext &ctx) {
+ if (!ProcessingResult.IsFatal) {
+ SendReadSessionsInfoToBalancer(ctx);
+ for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) {
+ const ui32 partitionIndex = partition.GetPartitionId();
const ui64 tabletId = partition.GetTabletId();
const bool inserted = PartitionToTablet.emplace(partitionIndex, tabletId).second;
Y_VERIFY(inserted);
@@ -516,11 +516,11 @@ void TPersQueueGetReadSessionsInfoTopicWorker::BootstrapImpl(const TActorContext
}
CreatePipeAndSend(tabletId, ctx, std::move(ev));
}
- } else {
- Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
+ } else {
+ Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
}
- if(WaitAllPipeEvents(ctx))
- return;
+ if(WaitAllPipeEvents(ctx))
+ return;
}
bool TPersQueueGetReadSessionsInfoTopicWorker::WaitAllPipeEvents(const TActorContext& ctx) {
@@ -565,7 +565,7 @@ bool TPersQueueGetReadSessionsInfoTopicWorker::HandleConnect(TEvTabletPipe::TEvC
if (ev->Status != NKikimrProto::OK) {
BalancerReplied = true;
if (ReadyToAnswer()) {
- Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
+ Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
}
} else {
TabletNodes[GetTabletId(ev)] = ev->ServerId.NodeId();
@@ -579,7 +579,7 @@ bool TPersQueueGetReadSessionsInfoTopicWorker::HandleDestroy(TEvTabletPipe::TEvC
}
BalancerReplied = true;
if (ReadyToAnswer()) {
- Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
+ Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
}
return true;
}
@@ -597,7 +597,7 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Handle(TEvPersQueue::TEvReadSessi
bool TPersQueueGetReadSessionsInfoTopicWorker::OnPipeEventsAreReady(const TActorContext& ctx) {
PipeEventsAreReady = true;
if (ReadyToAnswer()) {
- Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
+ Answer(ctx, ProcessingResult.Status, ProcessingResult.ErrorCode, ProcessingResult.Reason);
return true;
}
return false;
@@ -623,8 +623,8 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Answer(const TActorContext& ctx,
auto stat = response.MutableMetaResponse()->MutableCmdGetReadSessionsInfoResult();
auto topicRes = stat->AddTopicResult();
- topicRes->SetTopic(Name);
- SetErrorCode(topicRes, SchemeEntry);
+ topicRes->SetTopic(Name);
+ SetErrorCode(topicRes, SchemeEntry);
THashMap<ui32, ui32> partitionToResp;
ui32 index = 0;
if (BalancerResponse.Get() != nullptr) {
@@ -646,8 +646,8 @@ void TPersQueueGetReadSessionsInfoTopicWorker::Answer(const TActorContext& ctx,
ctx.Send(Parent, request.Release());
} else if (topicRes->GetErrorCode() == (ui32)NPersQueue::NErrorCode::OK) {
- for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) {
- const ui32 partitionIndex = partition.GetPartitionId();
+ for (const auto& partition : SchemeEntry.PQGroupInfo->Description.GetPartitions()) {
+ const ui32 partitionIndex = partition.GetPartitionId();
partitionToResp[partitionIndex] = index++;
auto res = topicRes->AddPartitionResult();
res->SetPartition(partitionIndex);
diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest.h b/ydb/core/client/server/msgbus_server_pq_metarequest.h
index deb696913f..1d56c28dc0 100644
--- a/ydb/core/client/server/msgbus_server_pq_metarequest.h
+++ b/ydb/core/client/server/msgbus_server_pq_metarequest.h
@@ -7,20 +7,20 @@ namespace NMsgBusProxy {
//
// GetTopicMetadata command
//
-//
+//
class TPersQueueGetTopicMetadataProcessor : public TPersQueueBaseRequestProcessor {
public:
TPersQueueGetTopicMetadataProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache);
private:
- THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
+ THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
};
-class TPersQueueGetTopicMetadataTopicWorker : public TReplierToParent<TTopicInfoBasedActor> {
+class TPersQueueGetTopicMetadataTopicWorker : public TReplierToParent<TTopicInfoBasedActor> {
public:
- TPersQueueGetTopicMetadataTopicWorker(const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name);
+ TPersQueueGetTopicMetadataTopicWorker(const TActorId& parent, const TSchemeEntry& topicEntry, const TString& name);
- void BootstrapImpl(const TActorContext& ctx) override;
+ void BootstrapImpl(const TActorContext& ctx) override;
void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override;
};
@@ -31,23 +31,23 @@ public:
class TPersQueueGetPartitionOffsetsProcessor : public TPersQueueBaseRequestProcessor {
public:
- TPersQueueGetPartitionOffsetsProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId);
+ TPersQueueGetPartitionOffsetsProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& metaCacheId);
private:
- THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
+ THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
private:
THashMap<TString, std::shared_ptr<THashSet<ui64>>> PartitionsToRequest;
};
-class TPersQueueGetPartitionOffsetsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> {
+class TPersQueueGetPartitionOffsetsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> {
public:
TPersQueueGetPartitionOffsetsTopicWorker(const TActorId& parent,
- const TSchemeEntry& topicEntry, const TString& name,
+ const TSchemeEntry& topicEntry, const TString& name,
const std::shared_ptr<THashSet<ui64>>& partitionsToRequest,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto);
- void BootstrapImpl(const TActorContext& ctx) override;
+ void BootstrapImpl(const TActorContext& ctx) override;
bool OnPipeEventsAreReady(const TActorContext& ctx) override;
void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override;
@@ -66,21 +66,21 @@ public:
TPersQueueGetPartitionStatusProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache);
private:
- THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
+ THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
private:
THashMap<TString, std::shared_ptr<THashSet<ui64>>> PartitionsToRequest;
};
-class TPersQueueGetPartitionStatusTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>> {
+class TPersQueueGetPartitionStatusTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvStatusResponse>> {
public:
TPersQueueGetPartitionStatusTopicWorker(const TActorId& parent,
- const TTopicInfoBasedActor::TSchemeEntry& topicEntry,
- const TString& name,
+ const TTopicInfoBasedActor::TSchemeEntry& topicEntry,
+ const TString& name,
const std::shared_ptr<THashSet<ui64>>& partitionsToRequest,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto);
- void BootstrapImpl(const TActorContext& ctx) override;
+ void BootstrapImpl(const TActorContext& ctx) override;
bool OnPipeEventsAreReady(const TActorContext& ctx) override;
void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override;
@@ -99,21 +99,21 @@ public:
TPersQueueGetPartitionLocationsProcessor(const NKikimrClient::TPersQueueRequest& request, const TActorId& schemeCache);
private:
- THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
+ THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
private:
THashMap<TString, std::shared_ptr<THashSet<ui64>>> PartitionsToRequest;
};
-class TPersQueueGetPartitionLocationsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>> {
+class TPersQueueGetPartitionLocationsTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvTabletPipe::TEvClientConnected>> {
public:
TPersQueueGetPartitionLocationsTopicWorker(const TActorId& parent,
- const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name,
+ const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name,
const std::shared_ptr<THashSet<ui64>>& partitionsToRequest,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto,
std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo);
- void BootstrapImpl(const TActorContext& ctx) override;
+ void BootstrapImpl(const TActorContext& ctx) override;
bool OnPipeEventsAreReady(const TActorContext& ctx) override;
void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override;
@@ -169,7 +169,7 @@ public:
private:
- THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
+ THolder<IActor> CreateTopicSubactor(const TSchemeEntry& topicEntry, const TString& name) override;
THolder<IActor> CreateSessionsSubactor(const THashMap<TString, TActorId>&& readSessions);
std::shared_ptr<IPersQueueGetReadSessionsInfoWorkerFactory> PQReadSessionsInfoWorkerFactory;
@@ -178,14 +178,14 @@ private:
};
-class TPersQueueGetReadSessionsInfoTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> {
+class TPersQueueGetReadSessionsInfoTopicWorker : public TReplierToParent<TPipesWaiterActor<TTopicInfoBasedActor, TEvPersQueue::TEvOffsetsResponse>> {
public:
TPersQueueGetReadSessionsInfoTopicWorker(const TActorId& parent,
- const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name,
+ const TTopicInfoBasedActor::TSchemeEntry& topicEntry, const TString& name,
const std::shared_ptr<const NKikimrClient::TPersQueueRequest>& requestProto,
std::shared_ptr<const TPersQueueBaseRequestProcessor::TNodesInfo> nodesInfo);
- void BootstrapImpl(const TActorContext& ctx) override;
+ void BootstrapImpl(const TActorContext& ctx) override;
void Answer(const TActorContext& ctx, EResponseStatus status, NPersQueue::NErrorCode::EErrorCode code, const TString& errorReason) override;
bool OnPipeEventsAreReady(const TActorContext& ctx) override;
diff --git a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
index 5271d241f8..237bba147b 100644
--- a/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
+++ b/ydb/core/client/server/msgbus_server_pq_metarequest_ut.cpp
@@ -25,9 +25,9 @@ namespace NKikimr {
namespace NMsgBusProxy {
using namespace testing;
-using namespace NSchemeCache;
+using namespace NSchemeCache;
-void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount);
+void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount);
void MakeEmptyTopic(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request);
void MakeDuplicatedTopic(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request);
void MakeDuplicatedPartition(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request);
@@ -314,7 +314,7 @@ protected:
void AssertFailedResponse(NPersQueue::NErrorCode::EErrorCode code, const THashSet<TString>& markers = {}, EResponseStatus status = MSTATUS_ERROR) {
const TEvPersQueue::TEvResponse* resp = GetResponse();
- Cerr << "Assert failed: Check response: " << resp->Record << Endl;
+ Cerr << "Assert failed: Check response: " << resp->Record << Endl;
UNIT_ASSERT(resp != nullptr);
UNIT_ASSERT_C(resp->Record.HasStatus(), "Response: " << resp->Record);
UNIT_ASSERT_UNEQUAL_C(resp->Record.GetStatus(), 1, "Response: " << resp->Record);
@@ -437,91 +437,91 @@ class TMessageBusServerPersQueueRequestCommonTest: public TMessageBusServerPersQ
public:
#define COMMON_TESTS_LIST() \
UNIT_TEST(HandlesTimeout) \
- UNIT_TEST(FailsOnFailedGetAllTopicsRequest) \
- UNIT_TEST(FailsOnBadRootStatusInGetNodeRequest) \
- UNIT_TEST(FailesOnNotATopic) \
+ UNIT_TEST(FailsOnFailedGetAllTopicsRequest) \
+ UNIT_TEST(FailsOnBadRootStatusInGetNodeRequest) \
+ UNIT_TEST(FailesOnNotATopic) \
UNIT_TEST(FailsOnNotOkStatusInGetNodeRequest) \
UNIT_TEST(FailsOnNoBalancerInGetNodeRequest) \
UNIT_TEST(FailsOnZeroBalancerTabletIdInGetNodeRequest) \
UNIT_TEST(FailsOnBalancerDescribeResultFailureWhenTopicsAreGivenExplicitly) \
/**/
- virtual NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) = 0;
+ virtual NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) = 0;
- void SetBalancerId(TSchemeCacheNavigate::TResultSet& resultSet, ui64 index, const TMaybe<ui64>& tabletId) {
- auto* newInfo = new TSchemeCacheNavigate::TPQGroupInfo(*resultSet[index].PQGroupInfo);
- if (tabletId.Defined()) {
- newInfo->Description.SetBalancerTabletID(*tabletId);
- } else {
- newInfo->Description.ClearBalancerTabletID();
+ void SetBalancerId(TSchemeCacheNavigate::TResultSet& resultSet, ui64 index, const TMaybe<ui64>& tabletId) {
+ auto* newInfo = new TSchemeCacheNavigate::TPQGroupInfo(*resultSet[index].PQGroupInfo);
+ if (tabletId.Defined()) {
+ newInfo->Description.SetBalancerTabletID(*tabletId);
+ } else {
+ newInfo->Description.ClearBalancerTabletID();
}
- resultSet[index].PQGroupInfo.Reset(newInfo);
- }
-
- TSchemeCacheNavigate::TEntry MakeEntry(
- ui64 topicId,
- TSchemeCacheNavigate::EStatus status = TSchemeCacheNavigate::EStatus::Ok,
- TSchemeCacheNavigate::EKind kind = TSchemeCacheNavigate::KindTopic,
- bool makePQDescription = true
- ) {
- TSchemeCacheNavigate::TEntry entry;
- entry.Status = status;
- entry.Kind = kind;
- entry.Path = {"Root", "PQ"};
- if (status != TSchemeCacheNavigate::EStatus::Ok || kind != TSchemeCacheNavigate::KindTopic
- || !makePQDescription
- ) {
- return entry;
+ resultSet[index].PQGroupInfo.Reset(newInfo);
+ }
+
+ TSchemeCacheNavigate::TEntry MakeEntry(
+ ui64 topicId,
+ TSchemeCacheNavigate::EStatus status = TSchemeCacheNavigate::EStatus::Ok,
+ TSchemeCacheNavigate::EKind kind = TSchemeCacheNavigate::KindTopic,
+ bool makePQDescription = true
+ ) {
+ TSchemeCacheNavigate::TEntry entry;
+ entry.Status = status;
+ entry.Kind = kind;
+ entry.Path = {"Root", "PQ"};
+ if (status != TSchemeCacheNavigate::EStatus::Ok || kind != TSchemeCacheNavigate::KindTopic
+ || !makePQDescription
+ ) {
+ return entry;
}
- auto *pqInfo = new TSchemeCacheNavigate::TPQGroupInfo();
- pqInfo->Kind = TSchemeCacheNavigate::KindTopic;
- auto &descr = pqInfo->Description;
- switch (topicId) {
- case 1:
- descr.SetName("topic1");
- descr.SetBalancerTabletID(MakeTabletID(0, 0, 100));
- descr.SetAlterVersion(42);
- descr.SetPartitionPerTablet(1);
- break;
- case 2:
- descr.SetName("topic2");
- descr.SetBalancerTabletID(MakeTabletID(0, 0, 200));
- descr.SetAlterVersion(5);
- descr.SetPartitionPerTablet(3);
- break;
- default:
- UNIT_FAIL("");
- }
- for (auto i = 0u; i < descr.GetPartitionPerTablet(); i++) {
- auto* part = descr.AddPartitions();
- part->SetPartitionId(i);
- part->SetTabletId(MakeTabletID(0, 0, topicId * 100 + 1 + i));
- }
- entry.PQGroupInfo.Reset(pqInfo);
- return entry;
- }
-
- TSchemeCacheNavigate::TResultSet MakeResultSet(bool valid = true) {
- TSchemeCacheNavigate::TResultSet resultSet;
- if (valid) {
- resultSet.emplace_back(std::move(
- MakeEntry(1)
- ));
- resultSet.emplace_back(std::move(
- MakeEntry(2)
- ));
+ auto *pqInfo = new TSchemeCacheNavigate::TPQGroupInfo();
+ pqInfo->Kind = TSchemeCacheNavigate::KindTopic;
+ auto &descr = pqInfo->Description;
+ switch (topicId) {
+ case 1:
+ descr.SetName("topic1");
+ descr.SetBalancerTabletID(MakeTabletID(0, 0, 100));
+ descr.SetAlterVersion(42);
+ descr.SetPartitionPerTablet(1);
+ break;
+ case 2:
+ descr.SetName("topic2");
+ descr.SetBalancerTabletID(MakeTabletID(0, 0, 200));
+ descr.SetAlterVersion(5);
+ descr.SetPartitionPerTablet(3);
+ break;
+ default:
+ UNIT_FAIL("");
+ }
+ for (auto i = 0u; i < descr.GetPartitionPerTablet(); i++) {
+ auto* part = descr.AddPartitions();
+ part->SetPartitionId(i);
+ part->SetTabletId(MakeTabletID(0, 0, topicId * 100 + 1 + i));
+ }
+ entry.PQGroupInfo.Reset(pqInfo);
+ return entry;
+ }
+
+ TSchemeCacheNavigate::TResultSet MakeResultSet(bool valid = true) {
+ TSchemeCacheNavigate::TResultSet resultSet;
+ if (valid) {
+ resultSet.emplace_back(std::move(
+ MakeEntry(1)
+ ));
+ resultSet.emplace_back(std::move(
+ MakeEntry(2)
+ ));
}
- return resultSet;
+ return resultSet;
}
void HandlesTimeout() {
- EXPECT_CALL(GetMockPQMetaCache(), HandleDescribeAllTopics(_, _)); // gets request and doesn't reply
+ EXPECT_CALL(GetMockPQMetaCache(), HandleDescribeAllTopics(_, _)); // gets request and doesn't reply
NKikimrClient::TPersQueueRequest request = MakeValidRequest();
RegisterActor(request);
Runtime->EnableScheduleForActor(Actor->SelfId());
TDispatchOptions options;
- options.FinalEvents.emplace_back([](IEventHandle& h) { return h.Type == TEvPqMetaCache::TEvDescribeAllTopicsRequest::EventType; }, 1);
+ options.FinalEvents.emplace_back([](IEventHandle& h) { return h.Type == TEvPqMetaCache::TEvDescribeAllTopicsRequest::EventType; }, 1);
Runtime->DispatchEvents(options);
Runtime->UpdateCurrentTime(Runtime->GetCurrentTime() + TDuration::MilliSeconds(90000 + 1));
@@ -530,8 +530,8 @@ public:
AssertFailedResponse(NPersQueue::NErrorCode::ERROR, {"Marker# PQ11", "Marker# PQ16"}, MSTATUS_TIMEOUT);
}
- void FailsOnFailedGetAllTopicsRequest() {
- GetMockPQMetaCache().SetAllTopicsAnswer(false);
+ void FailsOnFailedGetAllTopicsRequest() {
+ GetMockPQMetaCache().SetAllTopicsAnswer(false);
NKikimrClient::TPersQueueRequest request = MakeValidRequest();
RegisterActor(request);
@@ -541,61 +541,61 @@ public:
}
void FailsOnNotOkStatusInGetNodeRequest() {
- auto entry = MakeEntry(1);
- entry.Status = TSchemeCacheNavigate::EStatus::PathErrorUnknown;
+ auto entry = MakeEntry(1);
+ entry.Status = TSchemeCacheNavigate::EStatus::PathErrorUnknown;
- GetMockPQMetaCache().SetAllTopicsAnswer(true, TSchemeCacheNavigate::TResultSet{entry});
-
- NKikimrClient::TPersQueueRequest request = MakeValidRequest(1);
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, TSchemeCacheNavigate::TResultSet{entry});
+
+ NKikimrClient::TPersQueueRequest request = MakeValidRequest(1);
RegisterActor(request);
GrabResponseEvent();
- AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, "Marker# PQ150");
+ AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, "Marker# PQ150");
}
- void FailsOnBadRootStatusInGetNodeRequest() {
- auto resultSet = MakeResultSet();
- resultSet[0].Status = ESchemeStatus::RootUnknown;
+ void FailsOnBadRootStatusInGetNodeRequest() {
+ auto resultSet = MakeResultSet();
+ resultSet[0].Status = ESchemeStatus::RootUnknown;
+
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
- GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
-
NKikimrClient::TPersQueueRequest request = MakeValidRequest();
RegisterActor(request);
GrabResponseEvent();
- AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ1", "Marker# PQ14"});
+ AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ1", "Marker# PQ14"});
}
- void FailesOnNotATopic() {
- auto resultSet = MakeResultSet();
- resultSet[1].Kind = TSchemeCacheNavigate::KindPath;
- resultSet[1].PQGroupInfo = nullptr;
+ void FailesOnNotATopic() {
+ auto resultSet = MakeResultSet();
+ resultSet[1].Kind = TSchemeCacheNavigate::KindPath;
+ resultSet[1].PQGroupInfo = nullptr;
+
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
- GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
-
NKikimrClient::TPersQueueRequest request = MakeValidRequest();
RegisterActor(request);
GrabResponseEvent();
- AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ95", "Marker# PQ13"});
+ AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ95", "Marker# PQ13"});
}
void FailsOnNoBalancerInGetNodeRequest() {
- auto resultSet = MakeResultSet();
- SetBalancerId(resultSet, 0, Nothing());
- GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
+ auto resultSet = MakeResultSet();
+ SetBalancerId(resultSet, 0, Nothing());
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
NKikimrClient::TPersQueueRequest request = MakeValidRequest();
RegisterActor(request);
GrabResponseEvent();
- AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ93", "Marker# PQ193"});
+ AssertFailedResponse(NPersQueue::NErrorCode::UNKNOWN_TOPIC, {"Marker# PQ93", "Marker# PQ193"});
}
void FailsOnZeroBalancerTabletIdInGetNodeRequest() {
- auto resultSet = MakeResultSet();
- SetBalancerId(resultSet, 0, 0);
- GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
+ auto resultSet = MakeResultSet();
+ SetBalancerId(resultSet, 0, 0);
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, std::move(resultSet));
NKikimrClient::TPersQueueRequest request = MakeValidRequest();
RegisterActor(request);
@@ -605,17 +605,17 @@ public:
}
void FailsOnBalancerDescribeResultFailureWhenTopicsAreGivenExplicitly() {
- auto resultSet = MakeResultSet();
- resultSet[1].Status = TSchemeCacheNavigate::EStatus::LookupError;
- //SetBalancerId(resultSet, 1, 0);
- GetMockPQMetaCache().SetAllTopicsAnswer(true, resultSet);
+ auto resultSet = MakeResultSet();
+ resultSet[1].Status = TSchemeCacheNavigate::EStatus::LookupError;
+ //SetBalancerId(resultSet, 1, 0);
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, resultSet);
NKikimrClient::TPersQueueRequest request = MakeValidRequest();
RegisterActor(request);
GrabResponseEvent();
- AssertFailedResponse(NPersQueue::NErrorCode::ERROR, "Marker# PQ1");
-
+ AssertFailedResponse(NPersQueue::NErrorCode::ERROR, "Marker# PQ1");
+
}
// Implementation details for test with pipe disconnection for inheritance
@@ -627,7 +627,7 @@ public:
template <class TResponseEvent>
void HandlesPipeDisconnectionImpl(EDisconnectionMode disconnectionMode, std::function<void(EDisconnectionMode disconnectionMode)> dataValidationFunction, bool requestTheWholeTopic = false) {
- GetMockPQMetaCache().SetAllTopicsAnswer(true, std::forward<TSchemeCacheNavigate::TResultSet>(MakeResultSet()));
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, std::forward<TSchemeCacheNavigate::TResultSet>(MakeResultSet()));
PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}});
PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0});
@@ -725,14 +725,14 @@ public:
UNIT_TEST(SuccessfullyReplies)
UNIT_TEST_SUITE_END();
- NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
+ NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
NKikimrClient::TPersQueueRequest persQueueRequest;
persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN);
auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetTopicMetadata();
req.AddTopic("topic1");
- if (topicsCount > 1)
- req.AddTopic("topic2");
+ if (topicsCount > 1)
+ req.AddTopic("topic2");
return persQueueRequest;
}
@@ -746,7 +746,7 @@ public:
}
void SuccessfullyReplies() {
- GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
NKikimrClient::TPersQueueRequest req = MakeValidRequest();
RegisterActor(req);
@@ -777,13 +777,13 @@ public:
}
};
-void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount = 2) {
+void FillValidTopicRequest(NProtoBuf::RepeatedPtrField<::NKikimrClient::TPersQueueMetaRequest::TTopicRequest>& request, ui64 topicsCount = 2) {
{
auto& topic1 = *request.Add();
topic1.SetTopic("topic1");
}
- if (topicsCount > 1){
+ if (topicsCount > 1){
auto& topic2 = *request.Add();
topic2.SetTopic("topic2");
topic2.AddPartition(1);
@@ -819,12 +819,12 @@ public:
UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive)
UNIT_TEST_SUITE_END();
- NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
+ NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
NKikimrClient::TPersQueueRequest persQueueRequest;
persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN);
auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetPartitionLocations();
- FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount);
+ FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount);
return persQueueRequest;
}
@@ -856,7 +856,7 @@ public:
}
void SuccessfullyPassesResponsesFromTablets() {
- GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}});
PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0});
@@ -982,12 +982,12 @@ public:
UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive)
UNIT_TEST_SUITE_END();
- NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
+ NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
NKikimrClient::TPersQueueRequest persQueueRequest;
persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN);
auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetPartitionOffsets();
- FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount);
+ FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount);
return persQueueRequest;
}
@@ -1019,7 +1019,7 @@ public:
}
void SuccessfullyPassesResponsesFromTablets() {
- GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}});
PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0});
@@ -1143,12 +1143,12 @@ public:
UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive)
UNIT_TEST_SUITE_END();
- NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
+ NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
NKikimrClient::TPersQueueRequest persQueueRequest;
persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN);
auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetPartitionStatus();
- FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount);
+ FillValidTopicRequest(*req.MutableTopicRequest(), topicsCount);
return persQueueRequest;
}
@@ -1180,7 +1180,7 @@ public:
}
void SuccessfullyPassesResponsesFromTablets() {
- GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}});
PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0});
@@ -1296,15 +1296,15 @@ public:
UNIT_TEST(HandlesPipeDisconnection_AnswerDoesNotArrive)
UNIT_TEST_SUITE_END();
- NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
+ NKikimrClient::TPersQueueRequest MakeValidRequest(ui64 topicsCount = 2) override {
NKikimrClient::TPersQueueRequest persQueueRequest;
persQueueRequest.SetTicket("client_id@" BUILTIN_ACL_DOMAIN);
auto& req = *persQueueRequest.MutableMetaRequest()->MutableCmdGetReadSessionsInfo();
req.SetClientId("client_id");
req.AddTopic("topic1");
- if (topicsCount > 1)
- req.AddTopic("topic2");
+ if (topicsCount > 1)
+ req.AddTopic("topic2");
return persQueueRequest;
}
@@ -1327,7 +1327,7 @@ public:
}
void SuccessfullyPassesResponsesFromTablets() {
- GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
+ GetMockPQMetaCache().SetAllTopicsAnswer(true, MakeResultSet());
PrepareBalancer("topic1", MakeTabletID(0, 0, 100), {{1, MakeTabletID(0, 0, 101)}});
PreparePQTablet("topic1", MakeTabletID(0, 0, 101), {0});
diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp
index 63081b72a8..7f3146d21a 100644
--- a/ydb/core/client/server/msgbus_server_proxy.cpp
+++ b/ydb/core/client/server/msgbus_server_proxy.cpp
@@ -176,7 +176,7 @@ void TMessageBusServerProxy::Bootstrap(const TActorContext& ctx) {
auto cacheConfig = MakeIntrusive<NSchemeCache::TSchemeCacheConfig>(AppData(ctx), SchemeCacheCounters);
SchemeCache = ctx.ExecutorThread.RegisterActor(CreateSchemeBoardSchemeCache(cacheConfig.Get()));
- PqMetaCache = CreatePersQueueMetaCacheV2Id();
+ PqMetaCache = CreatePersQueueMetaCacheV2Id();
if (Server) {
Server->InitSession(ctx.ExecutorThread.ActorSystem, ctx.SelfID);
diff --git a/ydb/core/client/server/pq_metacache_v2_ut.cpp b/ydb/core/client/server/pq_metacache_v2_ut.cpp
index de48476681..f02fd420c9 100644
--- a/ydb/core/client/server/pq_metacache_v2_ut.cpp
+++ b/ydb/core/client/server/pq_metacache_v2_ut.cpp
@@ -1,5 +1,5 @@
-#include "msgbus_server_pq_metacache.h"
-
+#include "msgbus_server_pq_metacache.h"
+
#include <ydb/core/testlib/basics/runtime.h>
#include <ydb/core/testlib/test_client.h>
#include <ydb/core/testlib/test_pq_client.h>
@@ -7,274 +7,274 @@
#include <ydb/core/tx/scheme_board/cache.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-namespace NKikimr::NMsgBusProxy {
-using namespace NPqMetaCacheV2;
-using namespace Tests;
-using namespace NYdb;
-using namespace NYdb::NTable;
-using namespace NKikimr::NSchemeCache;
-namespace NTests {
-
-class TPqMetaCacheV2Test: public TTestBase {
- void SetUp() override {
- TTestBase::SetUp();
- TPortManager pm{};
- MainServerPort = pm.GetPort(2134);
- GrpcServerPort = pm.GetPort(2135);
- NKikimr::NPersQueueTests::TServerSettings settings = NKikimr::NPersQueueTests::PQSettings(MainServerPort);
- settings.SetDomainName("Root");
- settings.SetDomain(0);
- settings.SetUseRealThreads(true);
- Server = new NKikimr::Tests::TServer(settings);
-
- Server->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(GrpcServerPort));
- auto* runtime = Server->GetRuntime();
- //runtime->SetLogPriority(NKikimrServices::PERSQUEUE, NActors::NLog::PRI_DEBUG);
-// runtime->SetLogPriority(NKikimrServices::KQP_PROXY, NActors::NLog::PRI_DEBUG);
-// runtime->SetLogPriority(NKikimrServices::KQP_COMPILE_SERVICE, NActors::NLog::PRI_DEBUG);
- runtime->SetLogPriority(NKikimrServices::PQ_METACACHE, NActors::NLog::PRI_DEBUG);
- runtime->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_EMERG);
- runtime->SetLogPriority(NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, NActors::NLog::PRI_EMERG);
-
- Client = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(settings, GrpcServerPort);
-
- Client->InitRootScheme();
-
- NYdb::TDriverConfig driverCfg;
- driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << GrpcServerPort);
- YdbDriver = new NYdb::TDriver(driverCfg);
- TableClient = new NYdb::NTable::TTableClient(*YdbDriver);
- PQClient = new NYdb::NPersQueue::TPersQueueClient(*YdbDriver);
-
- Client->MkDir("/Root", "PQ");
- Client->MkDir("/Root/PQ", "Config");
- Client->MkDir("/Root/PQ/Config", "V2");
-
- auto tableDesc = TTableBuilder()
- .AddNullableColumn("path", EPrimitiveType::Utf8)
- .AddNullableColumn("dc", EPrimitiveType::Utf8)
- .SetPrimaryKeyColumns({"path", "dc"})
- .Build();
-
- auto session = CheckYdbResult(std::move(TableClient->CreateSession())).GetSession();
- CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Topics", std::move(tableDesc)));
-
- tableDesc = TTableBuilder()
- .AddNullableColumn("name", EPrimitiveType::String)
- .AddNullableColumn("version", EPrimitiveType::Uint64)
- .SetPrimaryKeyColumns({"name", "version"})
- .Build();
- CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Versions", std::move(tableDesc)));
-
- EdgeActorId = runtime->AllocateEdgeActor();
- auto config = MakeIntrusive<TSchemeCacheConfig>();
- config->Counters = new NMonitoring::TDynamicCounters;
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr::NMsgBusProxy {
+using namespace NPqMetaCacheV2;
+using namespace Tests;
+using namespace NYdb;
+using namespace NYdb::NTable;
+using namespace NKikimr::NSchemeCache;
+namespace NTests {
+
+class TPqMetaCacheV2Test: public TTestBase {
+ void SetUp() override {
+ TTestBase::SetUp();
+ TPortManager pm{};
+ MainServerPort = pm.GetPort(2134);
+ GrpcServerPort = pm.GetPort(2135);
+ NKikimr::NPersQueueTests::TServerSettings settings = NKikimr::NPersQueueTests::PQSettings(MainServerPort);
+ settings.SetDomainName("Root");
+ settings.SetDomain(0);
+ settings.SetUseRealThreads(true);
+ Server = new NKikimr::Tests::TServer(settings);
+
+ Server->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(GrpcServerPort));
+ auto* runtime = Server->GetRuntime();
+ //runtime->SetLogPriority(NKikimrServices::PERSQUEUE, NActors::NLog::PRI_DEBUG);
+// runtime->SetLogPriority(NKikimrServices::KQP_PROXY, NActors::NLog::PRI_DEBUG);
+// runtime->SetLogPriority(NKikimrServices::KQP_COMPILE_SERVICE, NActors::NLog::PRI_DEBUG);
+ runtime->SetLogPriority(NKikimrServices::PQ_METACACHE, NActors::NLog::PRI_DEBUG);
+ runtime->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_EMERG);
+ runtime->SetLogPriority(NKikimrServices::PERSQUEUE_CLUSTER_TRACKER, NActors::NLog::PRI_EMERG);
+
+ Client = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(settings, GrpcServerPort);
+
+ Client->InitRootScheme();
+
+ NYdb::TDriverConfig driverCfg;
+ driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << GrpcServerPort);
+ YdbDriver = new NYdb::TDriver(driverCfg);
+ TableClient = new NYdb::NTable::TTableClient(*YdbDriver);
+ PQClient = new NYdb::NPersQueue::TPersQueueClient(*YdbDriver);
+
+ Client->MkDir("/Root", "PQ");
+ Client->MkDir("/Root/PQ", "Config");
+ Client->MkDir("/Root/PQ/Config", "V2");
+
+ auto tableDesc = TTableBuilder()
+ .AddNullableColumn("path", EPrimitiveType::Utf8)
+ .AddNullableColumn("dc", EPrimitiveType::Utf8)
+ .SetPrimaryKeyColumns({"path", "dc"})
+ .Build();
+
+ auto session = CheckYdbResult(std::move(TableClient->CreateSession())).GetSession();
+ CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Topics", std::move(tableDesc)));
+
+ tableDesc = TTableBuilder()
+ .AddNullableColumn("name", EPrimitiveType::String)
+ .AddNullableColumn("version", EPrimitiveType::Uint64)
+ .SetPrimaryKeyColumns({"name", "version"})
+ .Build();
+ CheckYdbResult(session.CreateTable("/Root/PQ/Config/V2/Versions", std::move(tableDesc)));
+
+ EdgeActorId = runtime->AllocateEdgeActor();
+ auto config = MakeIntrusive<TSchemeCacheConfig>();
+ config->Counters = new NMonitoring::TDynamicCounters;
config->Roots.emplace_back(settings.Domain, 8716544, "Root");
- auto& domainsInfo = settings.AppConfig.GetDomainsConfig();
- for (const auto& domain : domainsInfo.GetDomain()) {
+ auto& domainsInfo = settings.AppConfig.GetDomainsConfig();
+ for (const auto& domain : domainsInfo.GetDomain()) {
config->Roots.emplace_back(domain.GetDomainId(), domain.GetSchemeRoot(), domain.GetName());
- }
- SchemeCacheId = runtime->Register(CreateSchemeBoardSchemeCache(config.Get()));
- MetaCacheId = runtime->Register(
- NPqMetaCacheV2::CreatePQMetaCache(config->Counters, TDuration::MilliSeconds(50))
- );
- runtime->EnableScheduleForActor(SchemeCacheId, true);
- runtime->EnableScheduleForActor(MetaCacheId, true);
- }
-
- struct TTopicInfo {
- TString Path;
- TString Cluster;
- bool DoCreate;
- };
-
- void AddTopics(const TVector<TTopicInfo>& topics, bool shiftVersion = true) {
- auto session = TableClient->GetSession().GetValueSync().GetSession();
- auto topicsPrepared = CheckYdbResult(session.PrepareDataQuery(UpsertTopicQuery)).GetQuery();
- TVector<TAsyncStatus> createTopicStatus;
- auto tx = session.BeginTransaction().GetValueSync().GetTransaction();
- auto txControl = TTxControl::Tx(tx);
- for (const auto& topic : topics) {
- auto fullPath = "/Root/PQ/" + ::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster);
- if (topic.DoCreate) {
- createTopicStatus.push_back(PQClient->CreateTopic(fullPath));
- }
- auto builder = topicsPrepared.GetParamsBuilder();
- {
- auto &param = builder.AddParam("$Path");
- param.Utf8(topic.Path);
- param.Build();
- }
- {
- auto &param = builder.AddParam("$Cluster");
- param.Utf8(topic.Cluster);
- param.Build();
- }
- CheckYdbResult(topicsPrepared.Execute(txControl, builder.Build()));
- }
- if (shiftVersion) {
- auto versionPrepared = CheckYdbResult(session.PrepareDataQuery(UpdateVersionQuery)).GetQuery();
- auto builder = versionPrepared.GetParamsBuilder();
- {
- auto &param = builder.AddParam("$Version");
- param.Uint64(++Version);
- param.Build();
- }
- CheckYdbResult(versionPrepared.Execute(txControl, builder.Build()));
- }
- CheckYdbResult(tx.Commit());
- for (auto& status : createTopicStatus) {
- CheckYdbResult(std::move(status));
- }
- }
-
- THolder<TEvPqNewMetaCache::TEvDescribeTopicsResponse> DoMetaCacheRequest(const TVector<TTopicInfo>& topicList) {
- IEventBase* ev;
- if (topicList.empty()) {
- ev = new TEvPqNewMetaCache::TEvDescribeAllTopicsRequest();
- } else {
- TVector<TString> topicNames;
- for (const auto &topic : topicList) {
- topicNames.emplace_back(std::move(::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster)));
- }
- ev = new TEvPqNewMetaCache::TEvDescribeTopicsRequest(topicNames);
- }
- auto handle = new IEventHandle(MetaCacheId, EdgeActorId, ev);
- Server->GetRuntime()->Send(handle);
- auto response = Server->GetRuntime()->GrabEdgeEvent<TEvPqNewMetaCache::TEvDescribeTopicsResponse>();
- return std::move(response);
- }
-
- void CheckTopicInfo(const TVector<TTopicInfo>& expected, TSchemeCacheNavigate* result) {
- ui64 i = 0;
- Cerr << "=== Got cache navigate response: \n" << result->ToString(NKikimr::NScheme::TTypeRegistry()) << Endl;
- Cerr << "=== Expect to have " << expected.size() << " records, got: " << result->ResultSet.size() << " records" << Endl;
- UNIT_ASSERT_VALUES_EQUAL_C(expected.size(), result->ResultSet.size(), "Result size mismatch");
- for (const auto& topic : expected) {
- Cerr << "===Check topic: " << topic.Path << ":" << topic.Cluster << Endl;
- auto& response = result->ResultSet[i++];
- if (topic.DoCreate) {
- UNIT_ASSERT(response.Status == TSchemeCacheNavigate::EStatus::Ok);
- UNIT_ASSERT(response.PQGroupInfo);
- UNIT_ASSERT(response.PQGroupInfo->Kind == NSchemeCache::TSchemeCacheNavigate::KindTopic);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(response.Status, TSchemeCacheNavigate::EStatus::PathErrorUnknown);
- }
- }
- }
-
- void WaitForVersion(ui64 version, TDuration timeout = TDuration::Seconds(5)) {
- ui64 currentVersion = 0;
- auto endTime = TInstant::Now() + timeout;
- auto* runtime = Server->GetRuntime();
- while (endTime > TInstant::Now()) {
- auto handle = new IEventHandle(MetaCacheId, EdgeActorId, new TEvPqNewMetaCache::TEvListTopicsRequest());
- runtime->Send(handle);
- auto response = runtime->GrabEdgeEvent<TEvPqNewMetaCache::TEvListTopicsResponse>();
- currentVersion = response->TopicsVersion;
- if (currentVersion >= version) {
- Cerr << "=== Got current topics version: " << currentVersion << Endl;
- return;
- }
- Sleep(TDuration::MilliSeconds(100));
- }
- UNIT_FAIL("Wait for topics version timed out");
- }
-
- void TestDescribeTopics() {
- auto topics = TVector<TTopicInfo>({
- {"topic1", "man", true},
- {"topic2", "man", true},
- {"topic3", "man", false}
- });
- AddTopics(topics);
- auto ev = DoMetaCacheRequest(topics);
- CheckTopicInfo(topics, ev->Result.Get());
- UNIT_ASSERT(ev);
- }
- void TestDescribeAllTopics() {
- auto topics = TVector<TTopicInfo>({
- {"topic1", "man", true},
- {"topic2", "man", true},
- {"topic3", "man", false}
- });
- AddTopics(topics);
- auto ev = DoMetaCacheRequest({});
- CheckTopicInfo(topics, ev->Result.Get());
- UNIT_ASSERT(ev);
- }
-
- void TestTopicsUpdate() {
- auto topics = TVector<TTopicInfo>({
- {"topic1", "man", true},
- {"topic2", "man", true},
- {"topic3", "man", false}
- });
- AddTopics(topics);
- CheckTopicInfo(topics, DoMetaCacheRequest(topics)->Result.Get());
- WaitForVersion(Version);
-
- TTopicInfo topic{"topic1", "sas", true};
- AddTopics({topic}, false);
-
- CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get());
-
- AddTopics({}, true);
- Cerr << "===Wait for version: " << Version << Endl;
- WaitForVersion(Version);
- topics.insert(topics.end() - 1, topic);
- CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get());
- }
-
- template<class T>
- T CheckYdbResult(NThreading::TFuture<T>&& asyncResult) {
- auto res = asyncResult.GetValueSync();
- UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
- return res;
- }
-
- TString GetUpsertTopicQuery(const TTopicInfo& info) {
- return TStringBuilder()
- << "--!syntax_v1\n"
- << "UPSERT INTO `/Root/PQ/Config/V2/Topics` "
- << "(path, dc) VALUES (" << info.Path << ", " << info.Cluster << ");";
- }
-
- UNIT_TEST_SUITE(TPqMetaCacheV2Test)
- UNIT_TEST(TestDescribeTopics)
- UNIT_TEST(TestDescribeAllTopics)
- UNIT_TEST(TestTopicsUpdate)
- UNIT_TEST_SUITE_END();
-private:
- ui16 MainServerPort;
- ui16 GrpcServerPort;
- THolder<Tests::TServer> Server;
-
- TActorId EdgeActorId;
- TActorId SchemeCacheId;
- TActorId MetaCacheId;
- ui64 Version = 1;
+ }
+ SchemeCacheId = runtime->Register(CreateSchemeBoardSchemeCache(config.Get()));
+ MetaCacheId = runtime->Register(
+ NPqMetaCacheV2::CreatePQMetaCache(config->Counters, TDuration::MilliSeconds(50))
+ );
+ runtime->EnableScheduleForActor(SchemeCacheId, true);
+ runtime->EnableScheduleForActor(MetaCacheId, true);
+ }
+
+ struct TTopicInfo {
+ TString Path;
+ TString Cluster;
+ bool DoCreate;
+ };
+
+ void AddTopics(const TVector<TTopicInfo>& topics, bool shiftVersion = true) {
+ auto session = TableClient->GetSession().GetValueSync().GetSession();
+ auto topicsPrepared = CheckYdbResult(session.PrepareDataQuery(UpsertTopicQuery)).GetQuery();
+ TVector<TAsyncStatus> createTopicStatus;
+ auto tx = session.BeginTransaction().GetValueSync().GetTransaction();
+ auto txControl = TTxControl::Tx(tx);
+ for (const auto& topic : topics) {
+ auto fullPath = "/Root/PQ/" + ::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster);
+ if (topic.DoCreate) {
+ createTopicStatus.push_back(PQClient->CreateTopic(fullPath));
+ }
+ auto builder = topicsPrepared.GetParamsBuilder();
+ {
+ auto &param = builder.AddParam("$Path");
+ param.Utf8(topic.Path);
+ param.Build();
+ }
+ {
+ auto &param = builder.AddParam("$Cluster");
+ param.Utf8(topic.Cluster);
+ param.Build();
+ }
+ CheckYdbResult(topicsPrepared.Execute(txControl, builder.Build()));
+ }
+ if (shiftVersion) {
+ auto versionPrepared = CheckYdbResult(session.PrepareDataQuery(UpdateVersionQuery)).GetQuery();
+ auto builder = versionPrepared.GetParamsBuilder();
+ {
+ auto &param = builder.AddParam("$Version");
+ param.Uint64(++Version);
+ param.Build();
+ }
+ CheckYdbResult(versionPrepared.Execute(txControl, builder.Build()));
+ }
+ CheckYdbResult(tx.Commit());
+ for (auto& status : createTopicStatus) {
+ CheckYdbResult(std::move(status));
+ }
+ }
+
+ THolder<TEvPqNewMetaCache::TEvDescribeTopicsResponse> DoMetaCacheRequest(const TVector<TTopicInfo>& topicList) {
+ IEventBase* ev;
+ if (topicList.empty()) {
+ ev = new TEvPqNewMetaCache::TEvDescribeAllTopicsRequest();
+ } else {
+ TVector<TString> topicNames;
+ for (const auto &topic : topicList) {
+ topicNames.emplace_back(std::move(::NPersQueue::BuildFullTopicName(topic.Path, topic.Cluster)));
+ }
+ ev = new TEvPqNewMetaCache::TEvDescribeTopicsRequest(topicNames);
+ }
+ auto handle = new IEventHandle(MetaCacheId, EdgeActorId, ev);
+ Server->GetRuntime()->Send(handle);
+ auto response = Server->GetRuntime()->GrabEdgeEvent<TEvPqNewMetaCache::TEvDescribeTopicsResponse>();
+ return std::move(response);
+ }
+
+ void CheckTopicInfo(const TVector<TTopicInfo>& expected, TSchemeCacheNavigate* result) {
+ ui64 i = 0;
+ Cerr << "=== Got cache navigate response: \n" << result->ToString(NKikimr::NScheme::TTypeRegistry()) << Endl;
+ Cerr << "=== Expect to have " << expected.size() << " records, got: " << result->ResultSet.size() << " records" << Endl;
+ UNIT_ASSERT_VALUES_EQUAL_C(expected.size(), result->ResultSet.size(), "Result size mismatch");
+ for (const auto& topic : expected) {
+ Cerr << "===Check topic: " << topic.Path << ":" << topic.Cluster << Endl;
+ auto& response = result->ResultSet[i++];
+ if (topic.DoCreate) {
+ UNIT_ASSERT(response.Status == TSchemeCacheNavigate::EStatus::Ok);
+ UNIT_ASSERT(response.PQGroupInfo);
+ UNIT_ASSERT(response.PQGroupInfo->Kind == NSchemeCache::TSchemeCacheNavigate::KindTopic);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(response.Status, TSchemeCacheNavigate::EStatus::PathErrorUnknown);
+ }
+ }
+ }
+
+ void WaitForVersion(ui64 version, TDuration timeout = TDuration::Seconds(5)) {
+ ui64 currentVersion = 0;
+ auto endTime = TInstant::Now() + timeout;
+ auto* runtime = Server->GetRuntime();
+ while (endTime > TInstant::Now()) {
+ auto handle = new IEventHandle(MetaCacheId, EdgeActorId, new TEvPqNewMetaCache::TEvListTopicsRequest());
+ runtime->Send(handle);
+ auto response = runtime->GrabEdgeEvent<TEvPqNewMetaCache::TEvListTopicsResponse>();
+ currentVersion = response->TopicsVersion;
+ if (currentVersion >= version) {
+ Cerr << "=== Got current topics version: " << currentVersion << Endl;
+ return;
+ }
+ Sleep(TDuration::MilliSeconds(100));
+ }
+ UNIT_FAIL("Wait for topics version timed out");
+ }
+
+ void TestDescribeTopics() {
+ auto topics = TVector<TTopicInfo>({
+ {"topic1", "man", true},
+ {"topic2", "man", true},
+ {"topic3", "man", false}
+ });
+ AddTopics(topics);
+ auto ev = DoMetaCacheRequest(topics);
+ CheckTopicInfo(topics, ev->Result.Get());
+ UNIT_ASSERT(ev);
+ }
+ void TestDescribeAllTopics() {
+ auto topics = TVector<TTopicInfo>({
+ {"topic1", "man", true},
+ {"topic2", "man", true},
+ {"topic3", "man", false}
+ });
+ AddTopics(topics);
+ auto ev = DoMetaCacheRequest({});
+ CheckTopicInfo(topics, ev->Result.Get());
+ UNIT_ASSERT(ev);
+ }
+
+ void TestTopicsUpdate() {
+ auto topics = TVector<TTopicInfo>({
+ {"topic1", "man", true},
+ {"topic2", "man", true},
+ {"topic3", "man", false}
+ });
+ AddTopics(topics);
+ CheckTopicInfo(topics, DoMetaCacheRequest(topics)->Result.Get());
+ WaitForVersion(Version);
+
+ TTopicInfo topic{"topic1", "sas", true};
+ AddTopics({topic}, false);
+
+ CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get());
+
+ AddTopics({}, true);
+ Cerr << "===Wait for version: " << Version << Endl;
+ WaitForVersion(Version);
+ topics.insert(topics.end() - 1, topic);
+ CheckTopicInfo(topics, DoMetaCacheRequest({})->Result.Get());
+ }
+
+ template<class T>
+ T CheckYdbResult(NThreading::TFuture<T>&& asyncResult) {
+ auto res = asyncResult.GetValueSync();
+ UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
+ return res;
+ }
+
+ TString GetUpsertTopicQuery(const TTopicInfo& info) {
+ return TStringBuilder()
+ << "--!syntax_v1\n"
+ << "UPSERT INTO `/Root/PQ/Config/V2/Topics` "
+ << "(path, dc) VALUES (" << info.Path << ", " << info.Cluster << ");";
+ }
+
+ UNIT_TEST_SUITE(TPqMetaCacheV2Test)
+ UNIT_TEST(TestDescribeTopics)
+ UNIT_TEST(TestDescribeAllTopics)
+ UNIT_TEST(TestTopicsUpdate)
+ UNIT_TEST_SUITE_END();
+private:
+ ui16 MainServerPort;
+ ui16 GrpcServerPort;
+ THolder<Tests::TServer> Server;
+
+ TActorId EdgeActorId;
+ TActorId SchemeCacheId;
+ TActorId MetaCacheId;
+ ui64 Version = 1;
std::shared_ptr<NKikimr::NPersQueueTests::TFlatMsgBusPQClient> Client;
-
+
std::shared_ptr<NYdb::TDriver> YdbDriver;
std::shared_ptr<NYdb::NTable::TTableClient> TableClient;
std::shared_ptr<NYdb::NPersQueue::TPersQueueClient> PQClient;
- TString UpsertTopicQuery = TStringBuilder()
- << "--!syntax_v1\n"
- << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; "
- << "UPSERT INTO `/Root/PQ/Config/V2/Topics`"
- << "(path, dc) VALUES ($Path, $Cluster);";
- TString UpdateVersionQuery = TStringBuilder()
- << "--!syntax_v1\n"
- << "DECLARE $Version as Uint64; "
- << "UPSERT INTO `/Root/PQ/Config/V2/Version`"
- << "(name, version) VALUES ('Topics', $Version);";
-
-
-};
-UNIT_TEST_SUITE_REGISTRATION(TPqMetaCacheV2Test);
-
-}// namespace NTests
+ TString UpsertTopicQuery = TStringBuilder()
+ << "--!syntax_v1\n"
+ << "DECLARE $Path as Utf8; DECLARE $Cluster as Utf8; "
+ << "UPSERT INTO `/Root/PQ/Config/V2/Topics`"
+ << "(path, dc) VALUES ($Path, $Cluster);";
+ TString UpdateVersionQuery = TStringBuilder()
+ << "--!syntax_v1\n"
+ << "DECLARE $Version as Uint64; "
+ << "UPSERT INTO `/Root/PQ/Config/V2/Version`"
+ << "(name, version) VALUES ('Topics', $Version);";
+
+
+};
+UNIT_TEST_SUITE_REGISTRATION(TPqMetaCacheV2Test);
+
+}// namespace NTests
} // namespace NKikimr::NMsgBusProxy
diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h
index 5f9c8cca20..faf1797413 100644
--- a/ydb/core/driver_lib/run/config.h
+++ b/ydb/core/driver_lib/run/config.h
@@ -47,7 +47,7 @@ union TBasicKikimrServicesMask {
bool EnableCms:1;
bool EnableNodeTable:1;
bool EnableGRpcProxyStatus:1;
- bool EnablePQ:1;
+ bool EnablePQ:1;
bool EnableSqs:1;
bool EnableConfigsDispatcher:1;
bool EnableSecurityServices:1;
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 73c167fbce..819c1478d1 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -1425,16 +1425,16 @@ void TMessageBusServicesInitializer::InitializeServices(NActors::TActorSystemSet
if (!IsServiceInitialized(setup, NMsgBusProxy::CreateMsgBusProxyId())
&& Config.HasMessageBusConfig() && Config.GetMessageBusConfig().GetStartBusProxy()) {
if (IActor *proxy = BusServer.CreateProxy()) {
- TDuration pqMetaRefresh = TDuration::MilliSeconds(appData->PQConfig.GetMetaCacheRefreshIntervalMilliSeconds());
- if (appData->PQConfig.GetEnabled()) {
- setup->LocalServices.emplace_back(
- NMsgBusProxy::CreatePersQueueMetaCacheV2Id(),
- TActorSetupCmd(
- NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(appData->Counters, pqMetaRefresh),
- TMailboxType::ReadAsFilled, appData->UserPoolId
- )
- );
- }
+ TDuration pqMetaRefresh = TDuration::MilliSeconds(appData->PQConfig.GetMetaCacheRefreshIntervalMilliSeconds());
+ if (appData->PQConfig.GetEnabled()) {
+ setup->LocalServices.emplace_back(
+ NMsgBusProxy::CreatePersQueueMetaCacheV2Id(),
+ TActorSetupCmd(
+ NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(appData->Counters, pqMetaRefresh),
+ TMailboxType::ReadAsFilled, appData->UserPoolId
+ )
+ );
+ }
}
if (IActor* traceService = BusServer.CreateMessageBusTraceService()) {
@@ -1496,9 +1496,9 @@ void TGRpcServicesInitializer::InitializeServices(NActors::TActorSystemSetup* se
if (appData->PQConfig.GetEnabled()) {
TDuration pqMetaRefresh = TDuration::Seconds(NMsgBusProxy::PQ_METACACHE_REFRESH_INTERVAL_SECONDS);
- IActor * cache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(
- appData->Counters, pqMetaRefresh
- );
+ IActor * cache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(
+ appData->Counters, pqMetaRefresh
+ );
Y_VERIFY(cache);
setup->LocalServices.emplace_back(
NMsgBusProxy::CreatePersQueueMetaCacheV2Id(),
@@ -2092,11 +2092,11 @@ TSqsServiceInitializer::TSqsServiceInitializer(const TKikimrRunConfig& runConfig
void TSqsServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
if (Config.GetSqsConfig().GetEnableSqs()) {
- ui32 grpcPort = 0;
- if (Config.HasGRpcConfig())
- grpcPort = Config.GetGRpcConfig().GetPort();
+ ui32 grpcPort = 0;
+ if (Config.HasGRpcConfig())
+ grpcPort = Config.GetGRpcConfig().GetPort();
{
- IActor* actor = NSQS::CreateSqsService(grpcPort);
+ IActor* actor = NSQS::CreateSqsService(grpcPort);
setup->LocalServices.emplace_back(
NSQS::MakeSqsServiceID(NodeId),
TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId));
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index b0c9e93416..a4f74aa4e0 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -606,7 +606,7 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) {
if (hasLegacy) {
// start legacy service
auto grpcService = new NGRpcProxy::TGRpcService();
- auto future = grpcService->Prepare(ActorSystem.Get(), NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), NMsgBusProxy::CreateMsgBusProxyId(), Counters);
+ auto future = grpcService->Prepare(ActorSystem.Get(), NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), NMsgBusProxy::CreateMsgBusProxyId(), Counters);
auto startCb = [grpcService](NThreading::TFuture<void> result) {
if (result.HasException()) {
try {
@@ -674,19 +674,19 @@ void TKikimrRunner::InitializeGRpc(const TKikimrRunConfig& runConfig) {
}
if (hasPQv1) {
- server.AddService(new NGRpcService::V1::TGRpcPersQueueService(ActorSystem.Get(), Counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId));
+ server.AddService(new NGRpcService::V1::TGRpcPersQueueService(ActorSystem.Get(), Counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId));
}
if (hasPQCD) {
// the service has its own flag since it should be capable of using custom grpc port
- const auto& pqcdConfig = AppData->PQClusterDiscoveryConfig;
- TMaybe<ui64> inflightLimit = Nothing();
- if (pqcdConfig.HasRequestInflightLimit() && pqcdConfig.GetRequestInflightLimit() > 0) {
- inflightLimit = pqcdConfig.GetRequestInflightLimit();
- }
- server.AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(
- ActorSystem.Get(), Counters, grpcRequestProxyId, inflightLimit
- ));
+ const auto& pqcdConfig = AppData->PQClusterDiscoveryConfig;
+ TMaybe<ui64> inflightLimit = Nothing();
+ if (pqcdConfig.HasRequestInflightLimit() && pqcdConfig.GetRequestInflightLimit() > 0) {
+ inflightLimit = pqcdConfig.GetRequestInflightLimit();
+ }
+ server.AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(
+ ActorSystem.Get(), Counters, grpcRequestProxyId, inflightLimit
+ ));
}
if (hasCms) {
@@ -1109,19 +1109,19 @@ void TKikimrRunner::InitializeActorSystem(
SqsHttp.Reset(new NSQS::TAsyncHttpServer(AppData->SqsConfig));
SqsHttp->Initialize(ActorSystem.Get(),
GetServiceCounters(AppData->Counters, "sqs"),
- GetServiceCounters(AppData->Counters, "ymq_public"),
+ GetServiceCounters(AppData->Counters, "ymq_public"),
AppData->UserPoolId);
}
}
if (runConfig.AppConfig.HasGRpcConfig()) {
if (const ui32 grpcPort = runConfig.AppConfig.GetGRpcConfig().GetPort()) {
- auto driverConfig = NYdb::TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort);
+ auto driverConfig = NYdb::TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort);
YdbDriver.Reset(new NYdb::TDriver(driverConfig));
AppData->YdbDriver = YdbDriver.Get();
- }
- }
-
+ }
+ }
+
if (YqSharedResources) {
YqSharedResources->Init(ActorSystem.Get());
}
@@ -1439,8 +1439,8 @@ void TKikimrRunner::KikimrStop(bool graceful) {
if (YdbDriver) {
YdbDriver->Stop(true);
- }
-
+ }
+
if (Monitoring) {
Monitoring->Stop();
}
diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h
index 28b2683ba3..dfd93a24e7 100644
--- a/ydb/core/driver_lib/run/run.h
+++ b/ydb/core/driver_lib/run/run.h
@@ -44,7 +44,7 @@ protected:
THolder<NSQS::TAsyncHttpServer> SqsHttp;
THolder<NYdb::TDriver> YdbDriver;
-
+
THolder<NKqp::TKqpShutdownController> KqpShutdownController;
TIntrusivePtr<NInterconnect::TPollerThreads> PollerThreads;
diff --git a/ydb/core/persqueue/metering_typedefs.h b/ydb/core/persqueue/metering_typedefs.h
index e49e387a1b..0b05329dbe 100644
--- a/ydb/core/persqueue/metering_typedefs.h
+++ b/ydb/core/persqueue/metering_typedefs.h
@@ -1,18 +1,18 @@
-#pragma once
-#include <library/cpp/json/writer/json_value.h>
-
-//ToDo: Remove after switching to new metering service
-
-namespace NKikimr::NMetering {
-struct TMetricValue {
- ui64 Quantity;
-
- /** Start time or moment when event occurred. usage -> start of billing json */
- TInstant StartTime;
-
- /** 'tags' in billing json. Concatenated with MetricMeta MetricTags (conflicting keys ore overridden)*/
- THashMap <TString, NJson::TJsonValue> MetricTags;
-
- TInstant EndTime = TInstant::Zero();
-};
-}
+#pragma once
+#include <library/cpp/json/writer/json_value.h>
+
+//ToDo: Remove after switching to new metering service
+
+namespace NKikimr::NMetering {
+struct TMetricValue {
+ ui64 Quantity;
+
+ /** Start time or moment when event occurred. usage -> start of billing json */
+ TInstant StartTime;
+
+ /** 'tags' in billing json. Concatenated with MetricMeta MetricTags (conflicting keys ore overridden)*/
+ THashMap <TString, NJson::TJsonValue> MetricTags;
+
+ TInstant EndTime = TInstant::Zero();
+};
+}
diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp
index 11657f09c2..cc8e2419cf 100644
--- a/ydb/core/persqueue/partition.cpp
+++ b/ydb/core/persqueue/partition.cpp
@@ -4546,7 +4546,7 @@ void TPartition::HandleWrites(const TActorContext& ctx)
ProcessReserveRequests(ctx);
if (!haveData && !haveDrop && !haveCheckDisk) { //no data writed/deleted
- if (!Requests.empty()) { //there could be change ownership requests that
+ if (!Requests.empty()) { //there could be change ownership requests that
bool res = ProcessWrites(request.Get(), ctx);
Y_VERIFY(!res);
}
diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp
index 6a0de22df0..9937dbe561 100644
--- a/ydb/core/persqueue/pq_impl.cpp
+++ b/ydb/core/persqueue/pq_impl.cpp
@@ -9,10 +9,10 @@
#include <ydb/core/protos/counters_keyvalue.pb.h>
#include <ydb/core/metering/metering.h>
#include <ydb/core/tablet/tablet_counters.h>
-#include <library/cpp/json/json_writer.h>
+#include <library/cpp/json/json_writer.h>
+
+#include <util/generic/strbuf.h>
-#include <util/generic/strbuf.h>
-
//TODO: move this code to vieiwer
#include <ydb/core/tablet/tablet_counters_aggregator.h>
@@ -583,7 +583,7 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx)
Config = NewConfig;
FillMeteringParams(ctx);
-
+
if (!Config.PartitionsSize()) {
for (const auto partitionId : Config.GetPartitionIds()) {
Config.AddPartitions()->SetPartitionId(partitionId);
@@ -1406,10 +1406,10 @@ void TPersQueue::HandleWriteRequest(const ui64 responseCookie, const TActorId& p
const NKikimrClient::TPersQueuePartitionRequest& req, const TActorContext& ctx)
{
Y_VERIFY(req.CmdWriteSize());
- FlushMetrics(false, ctx); // To ensure hours' border;
- if (req.HasPutUnitsSize()) {
- CurrentPutUnitsQuantity += req.GetPutUnitsSize();
- }
+ FlushMetrics(false, ctx); // To ensure hours' border;
+ if (req.HasPutUnitsSize()) {
+ CurrentPutUnitsQuantity += req.GetPutUnitsSize();
+ }
TVector <TEvPQ::TEvWrite::TMsg> msgs;
@@ -1949,8 +1949,8 @@ bool TPersQueue::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TAc
void TPersQueue::HandleDie(const TActorContext& ctx)
{
- FlushMetrics(true, ctx);
-
+ FlushMetrics(true, ctx);
+
for (const auto& p : Partitions) {
ctx.Send(p.second.Actor, new TEvents::TEvPoisonPill());
}
@@ -2034,122 +2034,122 @@ void TPersQueue::HandleWakeup(const TActorContext& ctx) {
for (auto& g : groups) {
AggregateAndSendLabeledCountersFor(g, ctx);
}
- FlushMetrics(false, ctx);
+ FlushMetrics(false, ctx);
ctx.Schedule(TDuration::Seconds(5), new TEvents::TEvWakeup());
}
-TString TPersQueue::GetMeteringJson(
- const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags,
- ui64 quantity, const TString& quantityUnit,
- const TInstant& start, const TInstant& end, const TInstant& now
-) {
- TStringStream output;
- NJson::TJsonWriter writer(&output, false);
-
- writer.OpenMap();
-
- writer.Write("cloud_id", Config.GetYcCloudId());
- writer.Write("folder_id", Config.GetYcFolderId());
- writer.Write("resource_id", ResourceId);
+TString TPersQueue::GetMeteringJson(
+ const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags,
+ ui64 quantity, const TString& quantityUnit,
+ const TInstant& start, const TInstant& end, const TInstant& now
+) {
+ TStringStream output;
+ NJson::TJsonWriter writer(&output, false);
+
+ writer.OpenMap();
+
+ writer.Write("cloud_id", Config.GetYcCloudId());
+ writer.Write("folder_id", Config.GetYcFolderId());
+ writer.Write("resource_id", ResourceId);
writer.Write("id", TStringBuilder() << metricBillingId << "-" << Config.GetYdbDatabaseId() << "-" << TabletID() << "-" << start.MilliSeconds() << "-" << (++MeteringCounter));
-
- writer.Write("schema", schemeName);
-
- writer.OpenMap("tags");
- for (const auto& [tag, value] : tags) {
- writer.Write(tag, value);
- }
- writer.CloseMap(); // "tags"
-
- writer.OpenMap("usage");
- writer.Write("quantity", quantity);
- writer.Write("unit", quantityUnit);
- writer.Write("start", start.Seconds());
+
+ writer.Write("schema", schemeName);
+
+ writer.OpenMap("tags");
+ for (const auto& [tag, value] : tags) {
+ writer.Write(tag, value);
+ }
+ writer.CloseMap(); // "tags"
+
+ writer.OpenMap("usage");
+ writer.Write("quantity", quantity);
+ writer.Write("unit", quantityUnit);
+ writer.Write("start", start.Seconds());
writer.Write("finish", end.Seconds());
- writer.CloseMap(); // "usage"
-
- writer.OpenMap("labels");
+ writer.CloseMap(); // "usage"
+
+ writer.OpenMap("labels");
writer.Write("datastreams_stream_name", StreamName);
writer.Write("ydb_database", Config.GetYdbDatabaseId());
- writer.CloseMap(); // "labels"
-
- writer.Write("version", "v1");
- writer.Write("source_id", ToString(TabletID()));
- writer.Write("source_wt", now.Seconds());
- writer.CloseMap();
- writer.Flush();
- output << Endl;
- return output.Str();
-}
-
-void TPersQueue::FlushMetrics(bool force, const TActorContext &ctx) {
- if (!MeteringEnabled) {
- return;
- }
+ writer.CloseMap(); // "labels"
+
+ writer.Write("version", "v1");
+ writer.Write("source_id", ToString(TabletID()));
+ writer.Write("source_wt", now.Seconds());
+ writer.CloseMap();
+ writer.Flush();
+ output << Endl;
+ return output.Str();
+}
+
+void TPersQueue::FlushMetrics(bool force, const TActorContext &ctx) {
+ if (!MeteringEnabled) {
+ return;
+ }
if (Config.PartitionsSize() == 0)
return;
- auto now = ctx.Now();
- bool needFlushRequests = force, needFlushShards = force;
-
- auto checkFlushInterval = [&](const TInstant& lastFlush, bool& flag) {
- if ((now - lastFlush) >= MetricsFlushInterval) {
- flag = true;
- }
- };
- checkFlushInterval(RequestsMetricsLastFlush, needFlushRequests);
- checkFlushInterval(ShardsMetricsLastFlush, needFlushShards);
-
- auto requestsEndTime = now;
- if (now.Hours() > RequestsMetricsLastFlush.Hours()) {
- needFlushRequests = true;
- // If we jump over a hour edge, report requests metrics for a previous hour;
- requestsEndTime = TInstant::Hours(RequestsMetricsLastFlush.Hours() + 1);
- Y_VERIFY(requestsEndTime < now);
- } else {
- Y_VERIFY(now.Hours() == RequestsMetricsLastFlush.Hours());
- }
-
- if (needFlushRequests) {
- if (CurrentPutUnitsQuantity > 0) {
- auto record = GetMeteringJson(
+ auto now = ctx.Now();
+ bool needFlushRequests = force, needFlushShards = force;
+
+ auto checkFlushInterval = [&](const TInstant& lastFlush, bool& flag) {
+ if ((now - lastFlush) >= MetricsFlushInterval) {
+ flag = true;
+ }
+ };
+ checkFlushInterval(RequestsMetricsLastFlush, needFlushRequests);
+ checkFlushInterval(ShardsMetricsLastFlush, needFlushShards);
+
+ auto requestsEndTime = now;
+ if (now.Hours() > RequestsMetricsLastFlush.Hours()) {
+ needFlushRequests = true;
+ // If we jump over a hour edge, report requests metrics for a previous hour;
+ requestsEndTime = TInstant::Hours(RequestsMetricsLastFlush.Hours() + 1);
+ Y_VERIFY(requestsEndTime < now);
+ } else {
+ Y_VERIFY(now.Hours() == RequestsMetricsLastFlush.Hours());
+ }
+
+ if (needFlushRequests) {
+ if (CurrentPutUnitsQuantity > 0) {
+ auto record = GetMeteringJson(
"put_units", "yds.events.puts.v1", {}, CurrentPutUnitsQuantity, "put_events",
- RequestsMetricsLastFlush, requestsEndTime, now
- );
- NMetering::SendMeteringJson(ctx, record);
- }
- CurrentPutUnitsQuantity = 0;
- RequestsMetricsLastFlush = now;
- }
- if (needFlushShards) {
+ RequestsMetricsLastFlush, requestsEndTime, now
+ );
+ NMetering::SendMeteringJson(ctx, record);
+ }
+ CurrentPutUnitsQuantity = 0;
+ RequestsMetricsLastFlush = now;
+ }
+ if (needFlushShards) {
ui64 writeQuota = Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond();
ui64 reservedSpace = Config.GetPartitionConfig().GetLifetimeSeconds() * writeQuota;
ui64 consumersThroughput = Config.ReadRulesSize() * writeQuota;
ui64 numPartitions = Config.PartitionsSize();
- THashMap<TString, ui64> tags = {
- {"reserved_throughput_bps", writeQuota}, {"shard_enhanced_consumers_throughput", consumersThroughput},
- {"reserved_storage_bytes", reservedSpace}
- };
- auto makeShardsMetricsJson = [&](TInstant& end) {
- auto res = GetMeteringJson(
+ THashMap<TString, ui64> tags = {
+ {"reserved_throughput_bps", writeQuota}, {"shard_enhanced_consumers_throughput", consumersThroughput},
+ {"reserved_storage_bytes", reservedSpace}
+ };
+ auto makeShardsMetricsJson = [&](TInstant& end) {
+ auto res = GetMeteringJson(
"reserved_resources", "yds.resources.reserved.v1", tags,
numPartitions * (end - ShardsMetricsLastFlush).Seconds(), "second",
- ShardsMetricsLastFlush, end, now
- );
- ShardsMetricsLastFlush = end;
- return res;
- };
- auto interval = TInstant::Hours(ShardsMetricsLastFlush.Hours() + 1);
- while (interval < now) {
- NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(interval));
- interval += TDuration::Hours(1);
- }
- if (ShardsMetricsLastFlush < now) {
- NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(now));
- }
- Y_VERIFY(ShardsMetricsLastFlush == now);
- }
-}
-
+ ShardsMetricsLastFlush, end, now
+ );
+ ShardsMetricsLastFlush = end;
+ return res;
+ };
+ auto interval = TInstant::Hours(ShardsMetricsLastFlush.Hours() + 1);
+ while (interval < now) {
+ NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(interval));
+ interval += TDuration::Hours(1);
+ }
+ if (ShardsMetricsLastFlush < now) {
+ NMetering::SendMeteringJson(ctx, makeShardsMetricsJson(now));
+ }
+ Y_VERIFY(ShardsMetricsLastFlush == now);
+ }
+}
+
bool TPersQueue::HandleHook(STFUNC_SIG)
{
SetActivityType(NKikimrServices::TActivity::PERSQUEUE_ACTOR);
diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h
index 0203a544f2..bc3bfb10ba 100644
--- a/ydb/core/persqueue/pq_impl.h
+++ b/ydb/core/persqueue/pq_impl.h
@@ -80,14 +80,14 @@ class TPersQueue : public NKeyValue::TKeyValueFlat {
void ReadState(const NKikimrClient::TKeyValueResponse::TReadResult& read, const TActorContext& ctx);
void FillMeteringParams(const TActorContext& ctx);
- TString GetMeteringJson(const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags,
- ui64 quantity, const TString& quantityUnit,
- const TInstant& start, const TInstant& end, const TInstant& now);
-
- void FlushMetrics(bool force, const TActorContext& ctx);
- void FlushRequests(bool force, const TActorContext& ctx);
- void FlushShardData(bool force, const TActorContext& ctx);
-
+ TString GetMeteringJson(const TString& metricBillingId, const TString& schemeName, const THashMap<TString, ui64>& tags,
+ ui64 quantity, const TString& quantityUnit,
+ const TInstant& start, const TInstant& end, const TInstant& now);
+
+ void FlushMetrics(bool force, const TActorContext& ctx);
+ void FlushRequests(bool force, const TActorContext& ctx);
+ void FlushShardData(bool force, const TActorContext& ctx);
+
TMaybe<TEvPQ::TEvRegisterMessageGroup::TBody> MakeRegisterMessageGroup(
const NKikimrClient::TPersQueuePartitionRequest::TCmdRegisterMessageGroup& cmd,
NPersQueue::NErrorCode::EErrorCode& code, TString& error) const;
@@ -176,14 +176,14 @@ private:
THashMap<ui64, TAutoPtr<TResponseBuilder>> ResponseProxy;
NMetrics::TResourceMetrics *ResourceMetrics;
-
- bool MeteringEnabled = false;
- TDuration MetricsFlushInterval;
- TInstant ShardsMetricsLastFlush = TInstant::Zero();
- TInstant RequestsMetricsLastFlush = TInstant::Zero();
- ui64 CurrentPutUnitsQuantity = 0;
- TString ResourceId;
- TString StreamName;
+
+ bool MeteringEnabled = false;
+ TDuration MetricsFlushInterval;
+ TInstant ShardsMetricsLastFlush = TInstant::Zero();
+ TInstant RequestsMetricsLastFlush = TInstant::Zero();
+ ui64 CurrentPutUnitsQuantity = 0;
+ TString ResourceId;
+ TString StreamName;
ui64 MeteringCounter = 0;
};
diff --git a/ydb/core/persqueue/pq_ut.cpp b/ydb/core/persqueue/pq_ut.cpp
index ac6d98a236..13a6122910 100644
--- a/ydb/core/persqueue/pq_ut.cpp
+++ b/ydb/core/persqueue/pq_ut.cpp
@@ -1327,7 +1327,7 @@ Y_UNIT_TEST(TestWriteSplit) {
}, [&](const TString& dispatchName, std::function<void(TTestActorRuntime&)> setup, bool& activeZone) {
TFinalizer finalizer(tc);
tc.Prepare(dispatchName, setup, activeZone);
- activeZone = false;
+ activeZone = false;
tc.Runtime->SetScheduledLimit(200);
PQTabletPrepare(20000000, 100 * 1024 * 1024, 0, {{"user1", true}}, tc); //never delete
@@ -1336,10 +1336,10 @@ Y_UNIT_TEST(TestWriteSplit) {
TVector<std::pair<ui64, TString>> data;
data.push_back({1, TString{size, 'b'}});
data.push_back({2, TString{size, 'a'}});
- activeZone = PlainOrSoSlow(true, false);
+ activeZone = PlainOrSoSlow(true, false);
CmdWrite(0, "sourceIdx", data, tc, false, {}, false, "", -1, 40000);
RestartTablet(tc);
- activeZone = false;
+ activeZone = false;
PQGetPartInfo(40000, 40002, tc);
});
}
diff --git a/ydb/core/persqueue/pq_ut_slow.cpp b/ydb/core/persqueue/pq_ut_slow.cpp
index 744fd3ba03..51041c1045 100644
--- a/ydb/core/persqueue/pq_ut_slow.cpp
+++ b/ydb/core/persqueue/pq_ut_slow.cpp
@@ -49,7 +49,7 @@ Y_UNIT_TEST(TestWriteVeryBigMessage) {
data.push_back({2, TString{size, 'a'}});
CmdWrite(1, "sourceIdx", data, tc, false, {}, false, "", -1, 80000);
CmdWrite(0, "sourceIdx", data, tc, false, {}, false, "", -1, 0);
- activeZone = true;
+ activeZone = true;
PQGetPartInfo(so, 1, tc);
RestartTablet(tc);
PQGetPartInfo(so, 1, tc);
diff --git a/ydb/core/persqueue/read_balancer.cpp b/ydb/core/persqueue/read_balancer.cpp
index 0de3728122..4eb9c7206d 100644
--- a/ydb/core/persqueue/read_balancer.cpp
+++ b/ydb/core/persqueue/read_balancer.cpp
@@ -348,7 +348,7 @@ void TPersQueueReadBalancer::RespondWithACL(
const TActorContext &ctx) {
THolder<TEvPersQueue::TEvCheckACLResponse> res{new TEvPersQueue::TEvCheckACLResponse};
res->Record.SetTopic(Topic);
- res->Record.SetPath(Path);
+ res->Record.SetPath(Path);
res->Record.SetAccess(access);
res->Record.SetError(error);
ctx.Send(request->Sender, res.Release());
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index cee10b1e22..d64169d4fc 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -841,36 +841,36 @@ message TSqsConfig {
optional uint64 MinimumGarbageAgeSeconds = 64 [default = 3600];
optional bool MeteringByNetClassifierOnly = 65 [default = false];
-
- message TYcSearchEventsConfig {
- optional bool EnableYcSearch = 1 [default = false];
- oneof OutputMethod {
- string UnifiedAgentUri = 10;
- string OutputFileName = 11;
- }
- optional uint64 RescanIntervalSeconds = 3 [default = 60];
- optional uint64 ReindexIntervalSeconds = 4 [default = 14400];
- optional bool TenantMode = 5 [default = false];
- }
-
- message TOauthTokenConfig {
- required string TokenFile = 1;
- }
-
- message TJwtConfig {
- required string JwtFile = 1;
- optional string IamEndpoint = 2;
- }
-
- message TYdbAuthConfig {
- oneof LocalAuthConfig {
- TOauthTokenConfig OauthToken = 1;
- TJwtConfig Jwt = 2;
- }
- }
-
- optional TYcSearchEventsConfig YcSearchEventsConfig = 66;
- optional TYdbAuthConfig AuthConfig = 67;
+
+ message TYcSearchEventsConfig {
+ optional bool EnableYcSearch = 1 [default = false];
+ oneof OutputMethod {
+ string UnifiedAgentUri = 10;
+ string OutputFileName = 11;
+ }
+ optional uint64 RescanIntervalSeconds = 3 [default = 60];
+ optional uint64 ReindexIntervalSeconds = 4 [default = 14400];
+ optional bool TenantMode = 5 [default = false];
+ }
+
+ message TOauthTokenConfig {
+ required string TokenFile = 1;
+ }
+
+ message TJwtConfig {
+ required string JwtFile = 1;
+ optional string IamEndpoint = 2;
+ }
+
+ message TYdbAuthConfig {
+ oneof LocalAuthConfig {
+ TOauthTokenConfig OauthToken = 1;
+ TJwtConfig Jwt = 2;
+ }
+ }
+
+ optional TYcSearchEventsConfig YcSearchEventsConfig = 66;
+ optional TYdbAuthConfig AuthConfig = 67;
}
message TConfigsDispatcherConfig {
diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto
index b20743c665..849b292383 100644
--- a/ydb/core/protos/msgbus_pq.proto
+++ b/ydb/core/protos/msgbus_pq.proto
@@ -137,7 +137,7 @@ message TPersQueuePartitionRequest {
optional TCmdUpdateWriteTimestamp CmdUpdateWriteTimestamp = 17;
optional bool IsDirectWrite = 18 [default = false];
- optional uint64 PutUnitsSize = 19;
+ optional uint64 PutUnitsSize = 19;
}
message TPersQueueMetaRequest {
diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto
index 8a3b59d60b..7c85927449 100644
--- a/ydb/core/protos/pqconfig.proto
+++ b/ydb/core/protos/pqconfig.proto
@@ -26,11 +26,11 @@ message TPQConfig {
//32 => 1mb blobs at last level
optional uint32 MaxBlobSize = 4 [default = 8388608]; //8mb
optional uint32 ClustersUpdateTimeoutSec = 5 [default = 30];
-
- optional bool Enabled = 6 [default = false]; // Enable PQ proxies
-
- optional uint32 MetaCacheTimeoutSec = 7 [default = 30];
-
+
+ optional bool Enabled = 6 [default = false]; // Enable PQ proxies
+
+ optional uint32 MetaCacheTimeoutSec = 7 [default = 30];
+
optional uint32 MaxReadCookies = 8 [default = 100000];
optional bool CheckACL = 9 [default = false];
@@ -107,7 +107,7 @@ message TPQConfig {
optional string SourceIdTablePath = 33 [default = "/Root/PQ/SourceIdMeta2"];
repeated uint32 ValidWriteSpeedLimitsKbPerSec = 34;
-
+
message TReadMeteringConfig {
message TPricing {
optional string Name = 1 [default = "unknown"];
@@ -116,13 +116,13 @@ message TPQConfig {
optional TPricing UnknownPricing = 3;
}
- message TBillingMeteringConfig {
- optional bool Enabled = 1 [default = false];
- optional uint64 FlushIntervalSec = 2 [default = 30];
+ message TBillingMeteringConfig {
+ optional bool Enabled = 1 [default = false];
+ optional uint64 FlushIntervalSec = 2 [default = 30];
optional TReadMeteringConfig Read = 3;
- }
+ }
- optional TBillingMeteringConfig BillingMeteringConfig = 35;
+ optional TBillingMeteringConfig BillingMeteringConfig = 35;
optional NKikimrNodeLimits.TNodeLimitsConfig.TPersQueueNodeConfig PersQueueNodeConfig = 36;
@@ -139,9 +139,9 @@ message TPQConfig {
repeated TClientServiceType ClientServiceType = 39;
optional TClientServiceType DefaultClientServiceType = 40;
optional bool DisallowDefaultClientServiceType = 43 [default = false];
-
- optional uint32 MetaCacheRefreshIntervalMilliSeconds = 41 [default = 10000];
- optional bool MetaCacheSkipVersionCheck = 42 [default = false];
+
+ optional uint32 MetaCacheRefreshIntervalMilliSeconds = 41 [default = 10000];
+ optional bool MetaCacheSkipVersionCheck = 42 [default = false];
}
message TChannelProfile {
@@ -259,11 +259,11 @@ message TPQTabletConfig {
optional uint64 AbcId = 24;
optional string AbcSlug = 25;
-
- optional string YcCloudId = 26 [default = ""];
- optional string YcFolderId = 27 [default = ""];
- optional string YdbDatabaseId = 28 [default = ""];
- optional string YdbDatabasePath = 29 [default = ""];
+
+ optional string YcCloudId = 26 [default = ""];
+ optional string YcFolderId = 27 [default = ""];
+ optional string YdbDatabaseId = 28 [default = ""];
+ optional string YdbDatabasePath = 29 [default = ""];
message TKeyComponentSchema {
optional string Name = 1;
@@ -365,7 +365,7 @@ message TCheckACL {
message TCheckACLResponse {
optional EAccess Access = 1;
optional string Topic = 2;
- optional string Path = 3;
+ optional string Path = 3;
optional string Error = 4;
}
@@ -680,7 +680,7 @@ message TPQClusterDiscoveryConfig {
optional bool Enabled = 1;
optional uint32 TimedCountersUpdateIntervalSeconds = 2 [default = 15];
optional NKikimrNetClassifier.TNetData CloudNetData = 3;
- optional uint64 RequestInflightLimit = 4;
+ optional uint64 RequestInflightLimit = 4;
}
message TYdsNextToken {
diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto
index 9fe4288ebe..c17c8a7dc3 100644
--- a/ydb/core/protos/services.proto
+++ b/ydb/core/protos/services.proto
@@ -169,7 +169,7 @@ enum EServiceKikimr {
PERSQUEUE_READ_BALANCER = 448;
CHOOSE_PROXY = 444;
- LB_CONFIG_MANAGER = 445;
+ LB_CONFIG_MANAGER = 445;
TOKEN_BUILDER = 450;
TICKET_PARSER = 455;
diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make
index 5b5d91da0c..70bb65514c 100644
--- a/ydb/core/protos/ya.make
+++ b/ydb/core/protos/ya.make
@@ -7,7 +7,7 @@ OWNER(
vvvv
g:kikimr
)
-
+
IF (OS_WINDOWS)
NO_OPTIMIZE_PY_PROTOS()
ENDIF()
diff --git a/ydb/core/testlib/mock_pq_metacache.h b/ydb/core/testlib/mock_pq_metacache.h
index 60bd72be42..1280f87779 100644
--- a/ydb/core/testlib/mock_pq_metacache.h
+++ b/ydb/core/testlib/mock_pq_metacache.h
@@ -9,9 +9,9 @@
namespace NKikimr {
-using TEvPqMetaCache = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache;
-using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>;
-using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus;
+using TEvPqMetaCache = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache;
+using TPQGroupInfoPtr = TIntrusiveConstPtr<NSchemeCache::TSchemeCacheNavigate::TPQGroupInfo>;
+using ESchemeStatus = NSchemeCache::TSchemeCacheNavigate::EStatus;
// The functionality of this class is not full.
// So anyone is welcome to improve it.
class TMockPQMetaCache: public TActor<TMockPQMetaCache> {
@@ -23,57 +23,57 @@ public:
STFUNC(StateFunc) {
switch (ev->GetTypeRewrite()) {
- HFunc(TEvPqMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics);
- HFunc(TEvPqMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics);
+ HFunc(TEvPqMetaCache::TEvDescribeTopicsRequest, HandleDescribeTopics);
+ HFunc(TEvPqMetaCache::TEvDescribeAllTopicsRequest, HandleDescribeAllTopics);
default:
UNIT_FAIL_NONFATAL("Unexpected event to PQ metacache: " << ev->GetTypeRewrite());
}
}
- MOCK_METHOD(void, HandleDescribeTopics, (TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx), ());
- MOCK_METHOD(void, HandleDescribeAllTopics, (TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx), ());
- //MOCK_METHOD4(HandleDescribeAllTopics, void(const TString& topic, ui64 balancerTabletId, NMsgBusProxy::TEvPqMetaCache::TEvGetBalancerDescribe::TPtr& ev, const TActorContext& ctx));
+ MOCK_METHOD(void, HandleDescribeTopics, (TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx), ());
+ MOCK_METHOD(void, HandleDescribeAllTopics, (TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx), ());
+ //MOCK_METHOD4(HandleDescribeAllTopics, void(const TString& topic, ui64 balancerTabletId, NMsgBusProxy::TEvPqMetaCache::TEvGetBalancerDescribe::TPtr& ev, const TActorContext& ctx));
//
// Helpers
//
- void SetDescribeCustomTopicsAnswer(const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {}) {
- // ToDo - !!!
+ void SetDescribeCustomTopicsAnswer(const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {}) {
+ // ToDo - !!!
using namespace testing;
-// auto handle = [success, description](NMsgBusProxy::TEvPqMetaCache::TEvGetNode::TPtr& ev, const TActorContext& ctx) {
-// auto& req = ev->Get()->Request;
-// req->Description = description;
-// req->Succeded = success;
-// auto result = MakeHolder<NMsgBusProxy::TEvPqMetaCache::TEvGetNodeResult>(std::move(req));
-//
-// ctx.Send(ev->Sender, std::move(result));
-// };
- auto handle = [=](TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) {
- auto* result = new NSchemeCache::TSchemeCacheNavigate();
- result->ResultSet = resultSet;
- auto* response = new TEvPqMetaCache::TEvDescribeTopicsResponse(ev->Get()->Topics, result);
- ctx.Send(ev->Sender, response);
+// auto handle = [success, description](NMsgBusProxy::TEvPqMetaCache::TEvGetNode::TPtr& ev, const TActorContext& ctx) {
+// auto& req = ev->Get()->Request;
+// req->Description = description;
+// req->Succeded = success;
+// auto result = MakeHolder<NMsgBusProxy::TEvPqMetaCache::TEvGetNodeResult>(std::move(req));
+//
+// ctx.Send(ev->Sender, std::move(result));
+// };
+ auto handle = [=](TEvPqMetaCache::TEvDescribeTopicsRequest::TPtr& ev, const TActorContext& ctx) {
+ auto* result = new NSchemeCache::TSchemeCacheNavigate();
+ result->ResultSet = resultSet;
+ auto* response = new TEvPqMetaCache::TEvDescribeTopicsResponse(ev->Get()->Topics, result);
+ ctx.Send(ev->Sender, response);
};
- EXPECT_CALL(*this, HandleDescribeTopics(_, _))
+ EXPECT_CALL(*this, HandleDescribeTopics(_, _))
.WillOnce(Invoke(handle));
}
- void SetAllTopicsAnswer(
- bool success = true, const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {}
- ) {
+ void SetAllTopicsAnswer(
+ bool success = true, const NSchemeCache::TSchemeCacheNavigate::TResultSet& resultSet = {}
+ ) {
using namespace testing;
- auto handle = [=](TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) {
- auto* response = new TEvPqMetaCache::TEvDescribeAllTopicsResponse("/Root/PQ/");
- response->Success = success;
- auto* result = new NSchemeCache::TSchemeCacheNavigate();
- result->ResultSet = resultSet;
- response->Result.Reset(result);
- ctx.Send(ev->Sender, std::move(response));
+ auto handle = [=](TEvPqMetaCache::TEvDescribeAllTopicsRequest::TPtr& ev, const TActorContext& ctx) {
+ auto* response = new TEvPqMetaCache::TEvDescribeAllTopicsResponse("/Root/PQ/");
+ response->Success = success;
+ auto* result = new NSchemeCache::TSchemeCacheNavigate();
+ result->ResultSet = resultSet;
+ response->Result.Reset(result);
+ ctx.Send(ev->Sender, std::move(response));
};
- EXPECT_CALL(*this, HandleDescribeAllTopics(_, _))
+ EXPECT_CALL(*this, HandleDescribeAllTopics(_, _))
.WillOnce(Invoke(handle));
}
};
diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp
index 1d1335c6ae..d4907c26f1 100644
--- a/ydb/core/testlib/test_client.cpp
+++ b/ydb/core/testlib/test_client.cpp
@@ -281,7 +281,7 @@ namespace Tests {
auto future = grpcService->Prepare(
system,
- NMsgBusProxy::CreatePersQueueMetaCacheV2Id(),
+ NMsgBusProxy::CreatePersQueueMetaCacheV2Id(),
NMsgBusProxy::CreateMsgBusProxyId(),
counters
);
@@ -306,7 +306,7 @@ namespace Tests {
GRpcServer->AddService(new NGRpcService::TGRpcYdbTableService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::TGRpcYdbScriptingService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::TGRpcOperationService(system, counters, grpcRequestProxyId));
- GRpcServer->AddService(new NGRpcService::V1::TGRpcPersQueueService(system, counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId));
+ GRpcServer->AddService(new NGRpcService::V1::TGRpcPersQueueService(system, counters, NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::TGRpcPQClusterDiscoveryService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NKesus::TKesusGRpcService(system, counters, grpcRequestProxyId));
GRpcServer->AddService(new NGRpcService::TGRpcCmsService(system, counters, grpcRequestProxyId));
@@ -661,33 +661,33 @@ namespace Tests {
}
{
- if (Settings->PQConfig.GetEnabled() == true) {
- IActor *pqMetaCache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(
- new NMonitoring::TDynamicCounters(), TDuration::Seconds(1)
- );
-
- TActorId pqMetaCacheId = Runtime->Register(pqMetaCache, nodeIdx);
- Runtime->RegisterService(NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), pqMetaCacheId, nodeIdx);
- }
+ if (Settings->PQConfig.GetEnabled() == true) {
+ IActor *pqMetaCache = NMsgBusProxy::NPqMetaCacheV2::CreatePQMetaCache(
+ new NMonitoring::TDynamicCounters(), TDuration::Seconds(1)
+ );
+
+ TActorId pqMetaCacheId = Runtime->Register(pqMetaCache, nodeIdx);
+ Runtime->RegisterService(NMsgBusProxy::CreatePersQueueMetaCacheV2Id(), pqMetaCacheId, nodeIdx);
+ }
+ }
+
+ {
+ if (Settings->EnableMetering) {
+ THolder<TFileLogBackend> fileBackend;
+ try {
+ fileBackend = MakeHolder<TFileLogBackend>(Settings->MeteringFilePath);
+ auto meteringActor = NMetering::CreateMeteringWriter(std::move(fileBackend));
+ TActorId meteringId = Runtime->Register(meteringActor.Release(), nodeIdx);
+ Runtime->RegisterService(NMetering::MakeMeteringServiceID(), meteringId, nodeIdx);
+
+ } catch (const TFileError &ex) {
+ Cerr << "TMeteringWriterInitializer: failed to open file '" << Settings->MeteringFilePath << "': "
+ << ex.what() << Endl;
+ }
+ }
}
{
- if (Settings->EnableMetering) {
- THolder<TFileLogBackend> fileBackend;
- try {
- fileBackend = MakeHolder<TFileLogBackend>(Settings->MeteringFilePath);
- auto meteringActor = NMetering::CreateMeteringWriter(std::move(fileBackend));
- TActorId meteringId = Runtime->Register(meteringActor.Release(), nodeIdx);
- Runtime->RegisterService(NMetering::MakeMeteringServiceID(), meteringId, nodeIdx);
-
- } catch (const TFileError &ex) {
- Cerr << "TMeteringWriterInitializer: failed to open file '" << Settings->MeteringFilePath << "': "
- << ex.what() << Endl;
- }
- }
- }
-
- {
IActor* kesusService = NKesus::CreateKesusProxyService();
TActorId kesusServiceId = Runtime->Register(kesusService, nodeIdx);
Runtime->RegisterService(NKesus::MakeKesusProxyServiceId(), kesusServiceId, nodeIdx);
diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h
index e4283d3d70..2064752ab4 100644
--- a/ydb/core/testlib/test_client.h
+++ b/ydb/core/testlib/test_client.h
@@ -122,9 +122,9 @@ namespace Tests {
ui64 DomainPlanResolution = 0;
std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> PersQueueGetReadSessionsInfoWorkerFactory;
- bool EnableMetering = false;
- TString MeteringFilePath;
-
+ bool EnableMetering = false;
+ TString MeteringFilePath;
+
std::function<IActor*(const NKikimrProto::TAuthConfig&)> CreateTicketParser = NKikimr::CreateTicketParser;
std::shared_ptr<TGrpcServiceFactory> GrpcServiceFactory;
@@ -158,7 +158,7 @@ namespace Tests {
TServerSettings& SetKeepSnapshotTimeout(TDuration value) { KeepSnapshotTimeout = value; return *this; }
TServerSettings& SetChangesQueueItemsLimit(ui64 value) { ChangesQueueItemsLimit = value; return *this; }
TServerSettings& SetChangesQueueBytesLimit(ui64 value) { ChangesQueueBytesLimit = value; return *this; }
- TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; }
+ TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; }
TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory(
std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory
) {
@@ -252,7 +252,7 @@ namespace Tests {
const bool UseStoragePools;
THolder<TTestActorRuntime> Runtime;
- THolder<NYdb::TDriver> Driver;
+ THolder<NYdb::TDriver> Driver;
TIntrusivePtr<NBus::TBusMessageQueue> Bus;
const NBus::TBusServerSessionConfig BusServerSessionConfig; //BusServer hold const & on config
TAutoPtr<NMsgBusProxy::IMessageBusServer> BusServer;
diff --git a/ydb/core/testlib/test_pq_client.h b/ydb/core/testlib/test_pq_client.h
index 5211010e10..f059923250 100644
--- a/ydb/core/testlib/test_pq_client.h
+++ b/ydb/core/testlib/test_pq_client.h
@@ -53,8 +53,8 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro
}
Tests::TServerSettings settings(port, authConfig, pqConfig);
- settings.SetDomainName("Root").SetNodeCount(nodesCount);
-
+ settings.SetDomainName("Root").SetNodeCount(nodesCount);
+
TVector<NKikimrKqp::TKqpSetting> kqpSettings;
NKikimrKqp::TKqpSetting kqpSetting;
kqpSetting.SetName("_KqpQueryTimeout");
@@ -62,9 +62,9 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro
kqpSetting.SetValue(yql_timeout);
kqpSettings.push_back(kqpSetting);
settings.SetKqpSettings(kqpSettings);
- settings.PQClusterDiscoveryConfig.SetEnabled(true);
- settings.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1);
-
+ settings.PQClusterDiscoveryConfig.SetEnabled(true);
+ settings.PQClusterDiscoveryConfig.SetTimedCountersUpdateIntervalSeconds(1);
+
if (netDataFile)
settings.NetClassifierConfig.SetNetDataFilePath(netDataFile->Name());
@@ -72,7 +72,7 @@ inline Tests::TServerSettings PQSettings(ui16 port, ui32 nodesCount = 2, bool ro
}
const TString TopicPrefix = "/Root/PQ/";
-const static TString DEFAULT_SRC_IDS_PATH = "/Root/PQ/SourceIdMeta2";
+const static TString DEFAULT_SRC_IDS_PATH = "/Root/PQ/SourceIdMeta2";
struct TRequestCreatePQ {
@@ -137,7 +137,7 @@ struct TRequestCreatePQ {
config->MutablePartitionConfig()->SetSourceIdLifetimeSeconds(SourceIdLifetime);
config->MutablePartitionConfig()->SetSourceIdMaxCounts(SourceIdMaxCount);
config->MutablePartitionConfig()->SetLowWatermark(LowWatermark);
-
+
config->SetLocalDC(true);
auto codec = config->MutableCodecs();
@@ -161,9 +161,9 @@ struct TRequestCreatePQ {
config->AddReadRuleVersions(0);
config->AddConsumerCodecs()->AddIds(0);
}
- if (!ReadRules.empty()) {
- config->SetRequireAuthRead(true);
- }
+ if (!ReadRules.empty()) {
+ config->SetRequireAuthRead(true);
+ }
if (!User.empty()) {
auto rq = config->MutablePartitionConfig()->AddReadQuota();
rq->SetSpeedInBytesPerSecond(ReadSpeed);
@@ -456,12 +456,12 @@ enum class ETransport {
GRpc
};
-struct TPQTestClusterInfo {
- TString Balancer;
- bool Enabled;
+struct TPQTestClusterInfo {
+ TString Balancer;
+ bool Enabled;
ui64 Weight = 1000;
-};
-
+};
+
static THashMap<TString, TPQTestClusterInfo> DEFAULT_CLUSTERS_LIST = {
{"dc1", {"localhost", true}},
{"dc2", {"dc2.logbroker.yandex.net", true}}
@@ -479,8 +479,8 @@ private:
const ui16 GRpcPort;
NClient::TKikimr Kikimr;
THolder<NYdb::TDriver> Driver;
- ui64 TopicsVersion = 0;
- bool UseConfigTables = true;
+ ui64 TopicsVersion = 0;
+ bool UseConfigTables = true;
void RunYqlSchemeQuery(TString query) {
auto tableClient = NYdb::NTable::TTableClient(*Driver);
@@ -515,9 +515,9 @@ private:
public:
- TFlatMsgBusPQClient(
- const Tests::TServerSettings& settings, ui16 grpc, TMaybe<TString> databaseName = Nothing()
- )
+ TFlatMsgBusPQClient(
+ const Tests::TServerSettings& settings, ui16 grpc, TMaybe<TString> databaseName = Nothing()
+ )
: TFlatMsgBusClient(settings)
, Settings(settings)
, GRpcPort(grpc)
@@ -535,24 +535,24 @@ public:
~TFlatMsgBusPQClient() {
Driver->Stop(true);
}
-
- void SetNoConfigMode() {
- UseConfigTables = false;
- }
-
- void FullInit(
- THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST,
- const TString& srcIdsPath = DEFAULT_SRC_IDS_PATH,
- const TString& localCluster = TString()
- ) {
- InitRoot();
- if (UseConfigTables) {
- InitSourceIds(srcIdsPath);
- InitDCs(clusters, localCluster);
- } else {
- InitSourceIds({});
- }
- }
+
+ void SetNoConfigMode() {
+ UseConfigTables = false;
+ }
+
+ void FullInit(
+ THashMap<TString, TPQTestClusterInfo> clusters = DEFAULT_CLUSTERS_LIST,
+ const TString& srcIdsPath = DEFAULT_SRC_IDS_PATH,
+ const TString& localCluster = TString()
+ ) {
+ InitRoot();
+ if (UseConfigTables) {
+ InitSourceIds(srcIdsPath);
+ InitDCs(clusters, localCluster);
+ } else {
+ InitSourceIds({});
+ }
+ }
void InitRoot() {
InitRootScheme();
MkDir("/Root", "PQ");
@@ -562,7 +562,7 @@ public:
return Driver.Get();
}
- void InitSourceIds(const TString& path = DEFAULT_SRC_IDS_PATH) {
+ void InitSourceIds(const TString& path = DEFAULT_SRC_IDS_PATH) {
TFsPath fsPath(path);
CreateTable(fsPath.Dirname(),
"Name: \"" + fsPath.Basename() + "\""
@@ -618,11 +618,11 @@ public:
weight Uint64,
PRIMARY KEY (name)
);
- CREATE TABLE [/Root/PQ/Config/V2/Topics] (
- path Utf8,
- dc Utf8,
- PRIMARY KEY (path, dc)
- );
+ CREATE TABLE [/Root/PQ/Config/V2/Topics] (
+ path Utf8,
+ dc Utf8,
+ PRIMARY KEY (path, dc)
+ );
)___");
RunYqlSchemeQuery(R"___(
@@ -633,34 +633,34 @@ public:
);
)___");
- TStringBuilder upsertClusters;
+ TStringBuilder upsertClusters;
upsertClusters << "UPSERT INTO [/Root/PQ/Config/V2/Cluster] (name, balancer, local, enabled, weight) VALUES ";
- bool first = true;
- for (auto& [cluster, info] : clusters) {
- bool isLocal = localCluster.empty() ? first : localCluster == cluster;
- if (!first)
- upsertClusters << ", ";
- upsertClusters << "(\"" << cluster << "\", \"" << info.Balancer << "\", " << (isLocal ? "true" : "false")
+ bool first = true;
+ for (auto& [cluster, info] : clusters) {
+ bool isLocal = localCluster.empty() ? first : localCluster == cluster;
+ if (!first)
+ upsertClusters << ", ";
+ upsertClusters << "(\"" << cluster << "\", \"" << info.Balancer << "\", " << (isLocal ? "true" : "false")
<< ", " << (info.Enabled ? "true" : "false") << ", " << info.Weight << ")";
- first = false;
- }
- upsertClusters << ";\n";
- TString clustersStr = clusters.empty() ? "" : TString(upsertClusters);
- Cerr << "=== Init DC: " << clustersStr << Endl;
+ first = false;
+ }
+ upsertClusters << ";\n";
+ TString clustersStr = clusters.empty() ? "" : TString(upsertClusters);
+ Cerr << "=== Init DC: " << clustersStr << Endl;
RunYqlDataQuery(clustersStr + R"___(
- UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Cluster", 1);
- UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Topics", 0);
+ UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Cluster", 1);
+ UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES ("Topics", 0);
)___");
}
- void UpdateDcEnabled(const TString& name, bool enabled) {
- TStringBuilder query;
- query << "UPDATE [/Root/PQ/Config/V2/Cluster] SET enabled = " << (enabled ? "true" : "false")
+ void UpdateDcEnabled(const TString& name, bool enabled) {
+ TStringBuilder query;
+ query << "UPDATE [/Root/PQ/Config/V2/Cluster] SET enabled = " << (enabled ? "true" : "false")
<< " where name = \"" << name << "\";";
- Cerr << "===Update clusters: " << query << Endl;
+ Cerr << "===Update clusters: " << query << Endl;
RunYqlDataQuery(query);
- }
-
+ }
+
TPQTestClusterInfo GetDcInfo(const TString& name) {
TStringBuilder query;
query << "SELECT balancer, enabled, weight FROM [/Root/PQ/Config/V2/Cluster] where name = \"" << name << "\";";
@@ -736,9 +736,9 @@ public:
TAutoPtr<NBus::TBusMessage> reply;
NBus::EMessageStatus status = SyncCall(request, reply);
- Cerr << "Topic created - response: " << PrintResult<NMsgBusProxy::TBusResponse>(reply.Get()) << Endl;
+ Cerr << "Topic created - response: " << PrintResult<NMsgBusProxy::TBusResponse>(reply.Get()) << Endl;
if (status != NBus::MESSAGE_OK)
- return 0;
+ return 0;
UNIT_ASSERT_VALUES_EQUAL(status, NBus::MESSAGE_OK);
const NMsgBusProxy::TBusResponse* response = dynamic_cast<NMsgBusProxy::TBusResponse*>(reply.Get());
UNIT_ASSERT(response);
@@ -759,7 +759,7 @@ public:
if (actualSize != cacheSize)
return 0;
}
- Cerr << "=== Topic created, have version: " << topicInfo.GetConfig().GetVersion() << Endl;
+ Cerr << "=== Topic created, have version: " << topicInfo.GetConfig().GetVersion() << Endl;
return topicInfo.GetConfig().GetVersion();
}
@@ -807,12 +807,12 @@ public:
TAutoPtr<NBus::TBusMessage> reply;
NBus::EMessageStatus status = SyncCall(request, reply);
- Cerr << "Topic deleted got reply with status: " << status << Endl;
+ Cerr << "Topic deleted got reply with status: " << status << Endl;
if (status != NBus::MESSAGE_OK)
return false;
const NMsgBusProxy::TBusResponse* response = dynamic_cast<NMsgBusProxy::TBusResponse*>(reply.Get());
- if (response == nullptr)
- return false;
+ if (response == nullptr)
+ return false;
Cerr << "TopicDeleted response " << response->Record << "\n";
UNIT_ASSERT(response);
UNIT_ASSERT(response->Record.HasErrorCode());
@@ -837,7 +837,7 @@ public:
}
void CreateConsumer(const TString& oldName) {
- auto name = NPersQueue::ConvertOldConsumerName(oldName);
+ auto name = NPersQueue::ConvertOldConsumerName(oldName);
RunYqlSchemeQuery("CREATE TABLE [/Root/PQ/" + name + "] (" + "Topic Utf8, Partition Uint32, Offset Uint64, PRIMARY KEY (Topic,Partition) );");
}
@@ -845,35 +845,35 @@ public:
NACLib::TDiffACL acl;
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::ReadAttributes, subj);
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::WriteAttributes, subj);
- auto name = NPersQueue::ConvertOldConsumerName(oldName);
+ auto name = NPersQueue::ConvertOldConsumerName(oldName);
auto pos = name.rfind("/");
Y_VERIFY(pos != TString::npos);
auto pref = "/Root/PQ/" + name.substr(0, pos);
ModifyACL(pref, name.substr(pos + 1), acl.SerializeAsString());
}
-
+
void CreateTopicNoLegacy(const TString& name, ui32 partsCount, bool doWait = true, bool canWrite = true) {
- TString path = name;
- if (UseConfigTables) {
- path = TStringBuilder() << "/Root/PQ/" << name;
- }
-
- auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver);
+ TString path = name;
+ if (UseConfigTables) {
+ path = TStringBuilder() << "/Root/PQ/" << name;
+ }
+
+ auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver);
auto settings = NYdb::NPersQueue::TCreateTopicSettings().PartitionsCount(partsCount).ClientWriteDisabled(!canWrite);
- Cerr << "===Create topic: " << path << Endl;
- auto res = pqClient.CreateTopic(path, settings);
- //ToDo - hack, cannot avoid legacy compat yet as PQv1 still uses RequestProcessor from core/client/server
- if (UseConfigTables) {
- AddTopic(name);
- }
- if (doWait) {
- res.Wait();
+ Cerr << "===Create topic: " << path << Endl;
+ auto res = pqClient.CreateTopic(path, settings);
+ //ToDo - hack, cannot avoid legacy compat yet as PQv1 still uses RequestProcessor from core/client/server
+ if (UseConfigTables) {
+ AddTopic(name);
+ }
+ if (doWait) {
+ res.Wait();
Cerr << "Create topic result: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << "\n";
UNIT_ASSERT(res.GetValue().IsSuccess());
- }
- }
-
+ }
+ }
+
void WaitTopicInit(const TString& topic) {
auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver);
do {
@@ -889,30 +889,30 @@ public:
} while (true);
}
- void CreateTopic(
- const TRequestCreatePQ& createRequest,
- bool doWait = true
- ) {
- const TInstant start = TInstant::Now();
-
- THolder<NMsgBusProxy::TBusPersQueue> request = createRequest.GetRequest();
-
- ui32 prevVersion = TopicCreated(createRequest.Topic);
- TAutoPtr<NBus::TBusMessage> reply;
- const NMsgBusProxy::TBusResponse* response = SendAndGetReply(request, reply);
- UNIT_ASSERT(response);
- UNIT_ASSERT_VALUES_EQUAL_C((ui32)response->Record.GetErrorCode(), (ui32)NPersQueue::NErrorCode::OK,
- "proxy failure");
-
- AddTopic(createRequest.Topic);
- while (doWait && TopicRealCreated(createRequest.Topic) != prevVersion + 1) {
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT);
+ void CreateTopic(
+ const TRequestCreatePQ& createRequest,
+ bool doWait = true
+ ) {
+ const TInstant start = TInstant::Now();
+
+ THolder<NMsgBusProxy::TBusPersQueue> request = createRequest.GetRequest();
+
+ ui32 prevVersion = TopicCreated(createRequest.Topic);
+ TAutoPtr<NBus::TBusMessage> reply;
+ const NMsgBusProxy::TBusResponse* response = SendAndGetReply(request, reply);
+ UNIT_ASSERT(response);
+ UNIT_ASSERT_VALUES_EQUAL_C((ui32)response->Record.GetErrorCode(), (ui32)NPersQueue::NErrorCode::OK,
+ "proxy failure");
+
+ AddTopic(createRequest.Topic);
+ while (doWait && TopicRealCreated(createRequest.Topic) != prevVersion + 1) {
+ Sleep(TDuration::MilliSeconds(500));
+ UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT);
+ }
+ while (doWait && TopicCreated(createRequest.Topic, prevVersion) != prevVersion + 1) {
+ Sleep(TDuration::MilliSeconds(500));
+ UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT);
}
- while (doWait && TopicCreated(createRequest.Topic, prevVersion) != prevVersion + 1) {
- Sleep(TDuration::MilliSeconds(500));
- UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT);
- }
}
void CreateTopic(
@@ -929,30 +929,30 @@ public:
ui64 sourceIdMaxCount = 6000000,
ui64 sourceIdLifetime = 86400
) {
- Y_VERIFY(name.StartsWith("rt3."));
-
- Cerr << "PQ Client: create topic: " << name << " with " << nParts << " partitions" << Endl;
+ Y_VERIFY(name.StartsWith("rt3."));
+
+ Cerr << "PQ Client: create topic: " << name << " with " << nParts << " partitions" << Endl;
auto request = TRequestCreatePQ(
- name, nParts, 0, lifetimeS, lowWatermark, writeSpeed, user, readSpeed, rr, important, mirrorFrom,
- sourceIdMaxCount, sourceIdLifetime
+ name, nParts, 0, lifetimeS, lowWatermark, writeSpeed, user, readSpeed, rr, important, mirrorFrom,
+ sourceIdMaxCount, sourceIdLifetime
);
- return CreateTopic(request);
- }
-
- void AlterTopicNoLegacy(const TString& name, ui32 nParts, ui64 lifetimeS = 86400) {
- TString path = name;
- if (!UseConfigTables) {
- path = TStringBuilder() << "/Root/PQ/" << name;
- }
- auto settings = NYdb::NPersQueue::TAlterTopicSettings().PartitionsCount(nParts);
- settings.RetentionPeriod(TDuration::Seconds(lifetimeS));
- auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver);
- auto res = pqClient.AlterTopic(path, settings);
- if (UseConfigTables) { // ToDo - legacy
- AlterTopic();
- }
- res.Wait();
- Cerr << "Alter topic (" << path << ") response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl;
+ return CreateTopic(request);
+ }
+
+ void AlterTopicNoLegacy(const TString& name, ui32 nParts, ui64 lifetimeS = 86400) {
+ TString path = name;
+ if (!UseConfigTables) {
+ path = TStringBuilder() << "/Root/PQ/" << name;
+ }
+ auto settings = NYdb::NPersQueue::TAlterTopicSettings().PartitionsCount(nParts);
+ settings.RetentionPeriod(TDuration::Seconds(lifetimeS));
+ auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver);
+ auto res = pqClient.AlterTopic(path, settings);
+ if (UseConfigTables) { // ToDo - legacy
+ AlterTopic();
+ }
+ res.Wait();
+ Cerr << "Alter topic (" << path << ") response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl;
}
void AlterTopic(
@@ -963,7 +963,7 @@ public:
bool fillPartitionConfig = false,
std::optional<NKikimrPQ::TMirrorPartitionConfig> mirrorFrom = {}
) {
- Y_VERIFY(name.StartsWith("rt3."));
+ Y_VERIFY(name.StartsWith("rt3."));
TRequestAlterPQ requestDescr(name, nParts, cacheSize, lifetimeS, fillPartitionConfig, mirrorFrom);
THolder<NMsgBusProxy::TBusPersQueue> request = requestDescr.GetRequest();
@@ -976,7 +976,7 @@ public:
response->Record.DebugString().c_str());
const TInstant start = TInstant::Now();
- AlterTopic();
+ AlterTopic();
while (TopicCreated(name, cacheSize) != prevVersion + 1) {
Sleep(TDuration::MilliSeconds(500));
UNIT_ASSERT(TInstant::Now() - start < ::DEFAULT_DISPATCH_TIMEOUT);
@@ -988,23 +988,23 @@ public:
}
- void DeleteTopicNoLegacy (const TString& name) {
- TString path = name;
- if (!UseConfigTables) {
- path = TStringBuilder() << "/Root/PQ/" << name;
- } else {
- RemoveTopic(name); // ToDo - also legacy
- }
- auto settings = NYdb::NPersQueue::TDropTopicSettings();
- auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver);
- auto res = pqClient.DropTopic(path, settings);
- res.Wait();
- Cerr << "Drop topic response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl;
- return;
- }
-
+ void DeleteTopicNoLegacy (const TString& name) {
+ TString path = name;
+ if (!UseConfigTables) {
+ path = TStringBuilder() << "/Root/PQ/" << name;
+ } else {
+ RemoveTopic(name); // ToDo - also legacy
+ }
+ auto settings = NYdb::NPersQueue::TDropTopicSettings();
+ auto pqClient = NYdb::NPersQueue::TPersQueueClient(*Driver);
+ auto res = pqClient.DropTopic(path, settings);
+ res.Wait();
+ Cerr << "Drop topic response: " << res.GetValue().IsSuccess() << " " << res.GetValue().GetIssues().ToString() << Endl;
+ return;
+ }
+
void DeleteTopic2(const TString& name, NPersQueue::NErrorCode::EErrorCode expectedStatus = NPersQueue::NErrorCode::OK, bool waitForTopicDeletion = true) {
- Y_VERIFY(name.StartsWith("rt3."));
+ Y_VERIFY(name.StartsWith("rt3."));
THolder<NMsgBusProxy::TBusPersQueue> request = TRequestDeletePQ{name}.GetRequest();
TAutoPtr<NBus::TBusMessage> reply;
@@ -1026,7 +1026,7 @@ public:
}
UNIT_ASSERT_C(i < 500, "Drop is taking too long"); //25 seconds
}
- RemoveTopic(name);
+ RemoveTopic(name);
const TInstant start = TInstant::Now();
while (waitForTopicDeletion && !TopicDeleted(name)) {
Sleep(TDuration::MilliSeconds(50));
@@ -1076,13 +1076,13 @@ public:
}
- void WriteToPQ(
- const TRequestWritePQ& writeRequest, const TString& data,
- const TString& ticket = "",
- ETransport transport = ETransport::MsgBus,
- NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK,
- NMsgBusProxy::EResponseStatus expectedOwnerStatus = NMsgBusProxy::MSTATUS_OK
- ) {
+ void WriteToPQ(
+ const TRequestWritePQ& writeRequest, const TString& data,
+ const TString& ticket = "",
+ ETransport transport = ETransport::MsgBus,
+ NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK,
+ NMsgBusProxy::EResponseStatus expectedOwnerStatus = NMsgBusProxy::MSTATUS_OK
+ ) {
TString cookie = GetOwnership({writeRequest.Topic, writeRequest.Partition}, expectedOwnerStatus);
@@ -1130,12 +1130,12 @@ public:
TVector<TString> Values;
};
- TReadDebugInfo ReadFromPQ(
- const TRequestReadPQ& readRequest, ui32 readCount,
- const TString& ticket = "",
- NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK,
- NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK
- ) {
+ TReadDebugInfo ReadFromPQ(
+ const TRequestReadPQ& readRequest, ui32 readCount,
+ const TString& ticket = "",
+ NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK,
+ NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK
+ ) {
THolder<NMsgBusProxy::TBusPersQueue> request = readRequest.GetRequest();
if (!ticket.empty()) {
request.Get()->Record.SetTicket(ticket);
@@ -1175,11 +1175,11 @@ public:
return ReadFromPQ({topic, partition, startOffset, count, "user"}, readCount, ticket);
}
- void SetClientOffsetPQ(
- const TRequestSetClientOffsetPQ& cmdRequest, const TString& ticket = "",
- NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK,
- NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK
- ) {
+ void SetClientOffsetPQ(
+ const TRequestSetClientOffsetPQ& cmdRequest, const TString& ticket = "",
+ NMsgBusProxy::EResponseStatus expectedStatus = NMsgBusProxy::MSTATUS_OK,
+ NPersQueue::NErrorCode::EErrorCode expectedError = NPersQueue::NErrorCode::OK
+ ) {
THolder<NMsgBusProxy::TBusPersQueue> request = cmdRequest.GetRequest();
if (!ticket.empty()) {
request.Get()->Record.SetTicket(ticket);
@@ -1372,51 +1372,51 @@ public:
GetPartLocation(topicsAndParts, resCount, ok);
GetPartStatus(topicsAndParts, resCount, ok);
}
-
-private:
- static TString GetAlterTopicsVersionQuery() {
- return "UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES (\"Topics\", $version);";
- }
-
-public:
- void AddTopic(const TString& topic) {
- return AddOrRemoveTopic(topic, true);
- }
-
- void RemoveTopic(const TString& topic) {
- return AddOrRemoveTopic(topic, false);
- }
-
- void AddOrRemoveTopic(const TString& topic, bool add) {
- TStringBuilder query;
- query << "DECLARE $version as Int64; DECLARE $path AS Utf8; DECLARE $cluster as Utf8; ";
- if (add) {
- query << "UPSERT INTO [/Root/PQ/Config/V2/Topics] (path, dc) VALUES ($path, $cluster); ";
- } else {
- query << "DELETE FROM [/Root/PQ/Config/V2/Topics] WHERE path = $path AND dc = $cluster; ";
- }
- query << GetAlterTopicsVersionQuery();
- NYdb::TParamsBuilder builder;
- auto params = builder
- .AddParam("$version").Int64(++TopicsVersion).Build()
- .AddParam("$path").Utf8(NPersQueue::GetTopicPath(topic)).Build()
- .AddParam("$cluster").Utf8(NPersQueue::GetDC(topic)).Build()
- .Build();
-
- Cerr << "===Run query:``" << query << "`` with topic = " << NPersQueue::GetTopicPath(topic) << ", dc = " << NPersQueue::GetDC(topic) << Endl;
- RunYqlDataQueryWithParams(query, params);
- Cerr << "===Query complete" << Endl;
- }
-
- void AlterTopic() {
- NYdb::TParamsBuilder builder;
- auto params = builder
- .AddParam("$version").Int64(++TopicsVersion).Build()
- .Build();
- TStringBuilder query;
- query << "DECLARE $version as Int64; " << GetAlterTopicsVersionQuery();
- RunYqlDataQueryWithParams(query, params);
- }
+
+private:
+ static TString GetAlterTopicsVersionQuery() {
+ return "UPSERT INTO [/Root/PQ/Config/V2/Versions] (name, version) VALUES (\"Topics\", $version);";
+ }
+
+public:
+ void AddTopic(const TString& topic) {
+ return AddOrRemoveTopic(topic, true);
+ }
+
+ void RemoveTopic(const TString& topic) {
+ return AddOrRemoveTopic(topic, false);
+ }
+
+ void AddOrRemoveTopic(const TString& topic, bool add) {
+ TStringBuilder query;
+ query << "DECLARE $version as Int64; DECLARE $path AS Utf8; DECLARE $cluster as Utf8; ";
+ if (add) {
+ query << "UPSERT INTO [/Root/PQ/Config/V2/Topics] (path, dc) VALUES ($path, $cluster); ";
+ } else {
+ query << "DELETE FROM [/Root/PQ/Config/V2/Topics] WHERE path = $path AND dc = $cluster; ";
+ }
+ query << GetAlterTopicsVersionQuery();
+ NYdb::TParamsBuilder builder;
+ auto params = builder
+ .AddParam("$version").Int64(++TopicsVersion).Build()
+ .AddParam("$path").Utf8(NPersQueue::GetTopicPath(topic)).Build()
+ .AddParam("$cluster").Utf8(NPersQueue::GetDC(topic)).Build()
+ .Build();
+
+ Cerr << "===Run query:``" << query << "`` with topic = " << NPersQueue::GetTopicPath(topic) << ", dc = " << NPersQueue::GetDC(topic) << Endl;
+ RunYqlDataQueryWithParams(query, params);
+ Cerr << "===Query complete" << Endl;
+ }
+
+ void AlterTopic() {
+ NYdb::TParamsBuilder builder;
+ auto params = builder
+ .AddParam("$version").Int64(++TopicsVersion).Build()
+ .Build();
+ TStringBuilder query;
+ query << "DECLARE $version as Int64; " << GetAlterTopicsVersionQuery();
+ RunYqlDataQueryWithParams(query, params);
+ }
};
} // namespace NPersQueueTests
diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h
index e811d71865..26a07897a1 100644
--- a/ydb/core/tx/scheme_cache/scheme_cache.h
+++ b/ydb/core/tx/scheme_cache/scheme_cache.h
@@ -263,10 +263,10 @@ struct TSchemeCacheNavigate {
: Instant(0)
{}
- explicit TSchemeCacheNavigate(ui64 instant)
- : Instant(instant)
- {}
-
+ explicit TSchemeCacheNavigate(ui64 instant)
+ : Instant(instant)
+ {}
+
TString ToString(const NScheme::TTypeRegistry& typeRegistry) const;
}; // TSchemeCacheNavigate
diff --git a/ydb/core/ymq/actor/action.h b/ydb/core/ymq/actor/action.h
index 04e319a5cd..1801c98d68 100644
--- a/ydb/core/ymq/actor/action.h
+++ b/ydb/core/ymq/actor/action.h
@@ -217,23 +217,23 @@ protected:
RLOG_SQS_ERROR("Request timeout. User [" << UserName_ << "] Queue [" << GetQueueName() << "] Action [" << Action_ << "]. State: { " << DumpState() << " }");
if (QueueCounters_) {
- INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second);
+ INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second);
} else if (UserCounters_) {
INC_COUNTER(UserCounters_, RequestTimeouts);
} else {
- TIntrusivePtrCntrCouple rootCounters {
- SqsCoreCounters_ ? SqsCoreCounters_ : GetSqsServiceCounters(AppData()->Counters, "core"),
- GetYmqPublicCounters(AppData()->Counters)
- };
- auto [userCounters, queueCounters] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, GetQueueName()));
- if (queueCounters.SqsCounters) {
- queueCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc();
- } else if (userCounters.SqsCounters) {
- userCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc();
+ TIntrusivePtrCntrCouple rootCounters {
+ SqsCoreCounters_ ? SqsCoreCounters_ : GetSqsServiceCounters(AppData()->Counters, "core"),
+ GetYmqPublicCounters(AppData()->Counters)
+ };
+ auto [userCounters, queueCounters] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, GetQueueName()));
+ if (queueCounters.SqsCounters) {
+ queueCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc();
+ } else if (userCounters.SqsCounters) {
+ userCounters.SqsCounters->GetCounter("RequestTimeouts", true)->Inc();
+ }
+ if (queueCounters.YmqCounters) {
+ queueCounters.YmqCounters->GetCounter("RequestTimeouts", true)->Inc();
}
- if (queueCounters.YmqCounters) {
- queueCounters.YmqCounters->GetCounter("RequestTimeouts", true)->Inc();
- }
}
MakeError(MutableErrorDesc(), NErrors::TIMEOUT);
@@ -260,30 +260,30 @@ protected:
void SendReplyAndDie() {
RLOG_SQS_TRACE("SendReplyAndDie from action actor " << Response_);
- auto actionCountersCouple = GetActionCounters();
+ auto actionCountersCouple = GetActionCounters();
auto* detailedCounters = UserCounters_ ? UserCounters_->GetDetailedCounters() : nullptr;
const size_t errors = ErrorsCount(Response_, detailedCounters ? &detailedCounters->APIStatuses : nullptr);
- if (actionCountersCouple.SqsCounters) {
+ if (actionCountersCouple.SqsCounters) {
if (errors) {
- ADD_COUNTER(actionCountersCouple.SqsCounters, Errors, errors);
+ ADD_COUNTER(actionCountersCouple.SqsCounters, Errors, errors);
} else {
- INC_COUNTER(actionCountersCouple.SqsCounters, Success);
+ INC_COUNTER(actionCountersCouple.SqsCounters, Success);
+ }
+ }
+ if (actionCountersCouple.YmqCounters) {
+ if (errors) {
+ ADD_COUNTER(actionCountersCouple.YmqCounters, Errors, errors);
+ } else {
+ INC_COUNTER(actionCountersCouple.YmqCounters, Success);
}
}
- if (actionCountersCouple.YmqCounters) {
- if (errors) {
- ADD_COUNTER(actionCountersCouple.YmqCounters, Errors, errors);
- } else {
- INC_COUNTER(actionCountersCouple.YmqCounters, Success);
- }
- }
FinishTs_ = TActivationContext::Now();
const TDuration workingDuration = GetRequestWorkingDuration();
RLOG_SQS_DEBUG("Request " << Action_ << " working duration: " << workingDuration.MilliSeconds() << "ms");
- if (actionCountersCouple.Defined()) {
+ if (actionCountersCouple.Defined()) {
const TDuration duration = GetRequestDuration();
- COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, Duration, duration.MilliSeconds());
- COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, WorkingDuration, workingDuration.MilliSeconds());
+ COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, Duration, duration.MilliSeconds());
+ COLLECT_HISTOGRAM_COUNTER_COUPLE(actionCountersCouple, WorkingDuration, workingDuration.MilliSeconds());
}
if (IsRequestSlow()) {
PrintSlowRequestWarning();
@@ -395,32 +395,32 @@ protected:
bool IsForbiddenPath(const TString& path) const {
return path.Contains("..") || path.Contains("//") || IsInternalResource(path);
}
- struct TActionCountersPack {
- TActionCounters* CoreCounters = nullptr;
- TActionCounters* YmqCounters = nullptr;
- };
+ struct TActionCountersPack {
+ TActionCounters* CoreCounters = nullptr;
+ TActionCounters* YmqCounters = nullptr;
+ };
- TCountersCouple<TActionCounters*> GetActionCounters() const {
- TCountersCouple<TActionCounters*> result{nullptr, nullptr};
+ TCountersCouple<TActionCounters*> GetActionCounters() const {
+ TCountersCouple<TActionCounters*> result{nullptr, nullptr};
if (IsActionForQueue(Action_) && QueueCounters_) {
if (IsActionForMessage(Action_) || QueueCounters_->NeedToShowDetailedCounters()) {
- result.SqsCounters = &QueueCounters_->SqsActionCounters[Action_];
+ result.SqsCounters = &QueueCounters_->SqsActionCounters[Action_];
}
} else if (IsActionForUser(Action_) && UserCounters_) {
if (UserCounters_->NeedToShowDetailedCounters()) {
- result.SqsCounters = &UserCounters_->SqsActionCounters[Action_];
+ result.SqsCounters = &UserCounters_->SqsActionCounters[Action_];
+ }
+ }
+ if (IsActionForQueueYMQ(Action_) && QueueCounters_) {
+ if (IsActionForMessage(Action_) || QueueCounters_->NeedToShowDetailedCounters()) {
+ result.YmqCounters = &QueueCounters_->YmqActionCounters[Action_];
+ }
+ } else if (IsActionForUserYMQ(Action_) && UserCounters_) {
+ if (UserCounters_->NeedToShowDetailedCounters()) {
+ result.YmqCounters = &UserCounters_->YmqActionCounters[Action_];
}
}
- if (IsActionForQueueYMQ(Action_) && QueueCounters_) {
- if (IsActionForMessage(Action_) || QueueCounters_->NeedToShowDetailedCounters()) {
- result.YmqCounters = &QueueCounters_->YmqActionCounters[Action_];
- }
- } else if (IsActionForUserYMQ(Action_) && UserCounters_) {
- if (UserCounters_->NeedToShowDetailedCounters()) {
- result.YmqCounters = &UserCounters_->YmqActionCounters[Action_];
- }
- }
- return result;
+ return result;
}
void RequestSchemeCache(const TString& path) {
@@ -696,39 +696,39 @@ private:
private:
void Start() {
- auto actionCountersCouple = GetActionCounters();
- if (actionCountersCouple.SqsCounters) {
+ auto actionCountersCouple = GetActionCounters();
+ if (actionCountersCouple.SqsCounters) {
if (IsActionForQueue(Action_) && QueueCounters_) {
- NeedReportSqsActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters();
+ NeedReportSqsActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters();
} else if (IsActionForUser(Action_) && UserCounters_) {
- NeedReportSqsActionInflyCounter = UserCounters_->NeedToShowDetailedCounters();
+ NeedReportSqsActionInflyCounter = UserCounters_->NeedToShowDetailedCounters();
}
- if (NeedReportSqsActionInflyCounter) {
- INC_COUNTER(actionCountersCouple.SqsCounters, Infly);
+ if (NeedReportSqsActionInflyCounter) {
+ INC_COUNTER(actionCountersCouple.SqsCounters, Infly);
+ }
+ }
+ if (actionCountersCouple.YmqCounters) {
+ if (IsActionForQueueYMQ(Action_) && QueueCounters_) {
+ NeedReportYmqActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters();
+ } else if (IsActionForUserYMQ(Action_) && UserCounters_) {
+ NeedReportYmqActionInflyCounter = UserCounters_->NeedToShowDetailedCounters();
+ }
+ if (NeedReportYmqActionInflyCounter) {
+ INC_COUNTER(actionCountersCouple.YmqCounters, Infly);
}
}
- if (actionCountersCouple.YmqCounters) {
- if (IsActionForQueueYMQ(Action_) && QueueCounters_) {
- NeedReportYmqActionInflyCounter = QueueCounters_->NeedToShowDetailedCounters();
- } else if (IsActionForUserYMQ(Action_) && UserCounters_) {
- NeedReportYmqActionInflyCounter = UserCounters_->NeedToShowDetailedCounters();
- }
- if (NeedReportYmqActionInflyCounter) {
- INC_COUNTER(actionCountersCouple.YmqCounters, Infly);
- }
- }
DoStart();
StartRequestWasCalled_ = true;
}
void Finish() {
- auto actionCounters = GetActionCounters();
- if (NeedReportSqsActionInflyCounter) {
- DEC_COUNTER(actionCounters.SqsCounters, Infly);
+ auto actionCounters = GetActionCounters();
+ if (NeedReportSqsActionInflyCounter) {
+ DEC_COUNTER(actionCounters.SqsCounters, Infly);
+ }
+ if (NeedReportYmqActionInflyCounter && actionCounters.YmqCounters) {
+ DEC_COUNTER(actionCounters.YmqCounters, Infly);
}
- if (NeedReportYmqActionInflyCounter && actionCounters.YmqCounters) {
- DEC_COUNTER(actionCounters.YmqCounters, Infly);
- }
if (StartRequestWasCalled_) {
DoFinish();
FinishRequestWasCalled_ = true;
@@ -756,7 +756,7 @@ protected:
TInstant StartTs_;
TInstant FinishTs_;
TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCoreCounters_; // Raw counters interface. Is is not prefered to use them
- TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_; // Raw counters interface. Is is not prefered to use them
+ TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_; // Raw counters interface. Is is not prefered to use them
TIntrusivePtr<TUserCounters> UserCounters_;
TIntrusivePtr<TQueueCounters> QueueCounters_;
TMaybe<TSqsEvents::TQueueAttributes> QueueAttributes_;
@@ -767,8 +767,8 @@ protected:
bool FinishRequestWasCalled_ = false;
TInstant QuotaRequestTs_;
TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> QuoterResources_;
- bool NeedReportSqsActionInflyCounter = false;
- bool NeedReportYmqActionInflyCounter = false;
+ bool NeedReportSqsActionInflyCounter = false;
+ bool NeedReportYmqActionInflyCounter = false;
TSchedulerCookieHolder TimeoutCookie_ = ISchedulerCookie::Make2Way();
NKikimrClient::TSqsRequest SourceSqsRequest_;
};
diff --git a/ydb/core/ymq/actor/delete_message.cpp b/ydb/core/ymq/actor/delete_message.cpp
index 7e11e5b061..50b59ddcfb 100644
--- a/ydb/core/ymq/actor/delete_message.cpp
+++ b/ydb/core/ymq/actor/delete_message.cpp
@@ -94,7 +94,7 @@ private:
// Calc metrics
const TDuration processingDuration = TActivationContext::Now() - lockTimestamp;
COLLECT_HISTOGRAM_COUNTER(QueueCounters_, ClientMessageProcessing_Duration, processingDuration.MilliSeconds());
- COLLECT_HISTOGRAM_COUNTER(QueueCounters_, client_processing_duration_milliseconds, processingDuration.MilliSeconds());
+ COLLECT_HISTOGRAM_COUNTER(QueueCounters_, client_processing_duration_milliseconds, processingDuration.MilliSeconds());
} catch (...) {
RLOG_SQS_WARN("Failed to process receipt handle " << entry.GetReceiptHandle() << ": " << CurrentExceptionMessage());
MakeError(resp, NErrors::RECEIPT_HANDLE_IS_INVALID);
@@ -104,7 +104,7 @@ private:
void ProcessAnswer(TDeleteMessageResponse* resp, const TSqsEvents::TEvDeleteMessageBatchResponse::TMessageResult& answer) {
switch (answer.Status) {
case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::OK: {
- INC_COUNTER_COUPLE(QueueCounters_, DeleteMessage_Count, deleted_count_per_second);
+ INC_COUNTER_COUPLE(QueueCounters_, DeleteMessage_Count, deleted_count_per_second);
break;
}
case TSqsEvents::TEvDeleteMessageBatchResponse::EDeleteMessageStatus::NotFound: {
diff --git a/ydb/core/ymq/actor/events.h b/ydb/core/ymq/actor/events.h
index 0e5948556d..9fff62c7cf 100644
--- a/ydb/core/ymq/actor/events.h
+++ b/ydb/core/ymq/actor/events.h
@@ -124,8 +124,8 @@ struct TSqsEvents {
EvGarbageSearchResult,
- EvCleanupQueryComplete,
-
+ EvCleanupQueryComplete,
+
EvEnd,
};
@@ -876,15 +876,15 @@ struct TSqsEvents {
bool Success = true;
THashMap<TString, TGarbageHint> GarbageHints;
};
- struct TEvCleanupQueryComplete : public NActors::TEventLocal<TEvCleanupQueryComplete, EvCleanupQueryComplete> {
- explicit TEvCleanupQueryComplete(const TString& name, ui64 type)
- : Name(name)
- , Type(type)
- {
- }
- TString Name;
- ui64 Type;
- };
+ struct TEvCleanupQueryComplete : public NActors::TEventLocal<TEvCleanupQueryComplete, EvCleanupQueryComplete> {
+ explicit TEvCleanupQueryComplete(const TString& name, ui64 type)
+ : Name(name)
+ , Type(type)
+ {
+ }
+ TString Name;
+ ui64 Type;
+ };
};
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/index_events_processor.cpp b/ydb/core/ymq/actor/index_events_processor.cpp
index 16ba23d32f..646e06420e 100644
--- a/ydb/core/ymq/actor/index_events_processor.cpp
+++ b/ydb/core/ymq/actor/index_events_processor.cpp
@@ -1,493 +1,493 @@
-#include "index_events_processor.h"
+#include "index_events_processor.h"
#include <ydb/public/sdk/cpp/client/ydb_types/ydb.h>
-
-namespace NKikimr::NSQS {
-using namespace NActors;
-using namespace NJson;
-using namespace NYdb;
-
-constexpr TDuration DEFAULT_RETRY_TIMEOUT = TDuration::Seconds(10);
-constexpr TDuration SHORT_RETRY_TIMEOUT = TDuration::Seconds(2);
-constexpr TDuration DEFAULT_QUERY_TIMEOUT = TDuration::Seconds(30);
-
-TSearchEventsProcessor::TSearchEventsProcessor(
- const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval,
- const TSimpleSharedPtr<NTable::TTableClient>& tableClient,
- IEventsWriterWrapper::TPtr eventsWriter,
- bool waitForWake
-)
- : RescanInterval(rescanInterval)
- , ReindexInterval(reindexInterval)
- , TableClient(tableClient)
- , EventsWriter(eventsWriter)
- , WaitForWake(waitForWake)
-{
- InitQueries(root);
-}
-
-void TSearchEventsProcessor::InitQueries(const TString& root) {
- auto getTableFullPath = [&](const TString& tableName) {
- if (!root.empty()) {
- return root + "/" + tableName;
- } else {
- return tableName;
- }
- };
- TString eventsTablePath = getTableFullPath(".Events");
-
- SelectQueuesQuery = TStringBuilder()
- << "--!syntax_v1\n"
- << "DECLARE $Account as Utf8; DECLARE $QueueName as Utf8; "
- << "SELECT Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId from `"
- << getTableFullPath(".Queues") << "` "
- << "WHERE Account > $Account OR (Account = $Account AND QueueName > $QueueName);";
-
- SelectEventsQuery = TStringBuilder()
- << "--!syntax_v1\n"
- << "SELECT Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId "
- << "FROM `"<< eventsTablePath << "`;";
-
- DeleteEventQuery = TStringBuilder()
- << "--!syntax_v1\n"
- << "DECLARE $Events AS List<Struct<Account:Utf8, QueueName:Utf8, EventType:Uint64>>;\n"
- << "$EventsSource = (SELECT item.Account AS Account, item.QueueName AS QueueName, item.EventType AS EventType \n"
- << "FROM (SELECT $Events AS events) FLATTEN LIST BY events as item);\n"
- << "DELETE FROM `" << eventsTablePath << "` ON SELECT * FROM $EventsSource;";
-}
-
-TSearchEventsProcessor::~TSearchEventsProcessor() {
- EventsWriter->Close();
-}
-
-void TSearchEventsProcessor::Bootstrap(const TActorContext& ctx) {
- Become(&TSearchEventsProcessor::StateFunc);
- if (!WaitForWake)
- ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
-}
-
-STFUNC(TSearchEventsProcessor::StateFunc) {
- switch (ev->GetTypeRewrite()) {
- HFunc(TEvWakeup, HandleWakeup);
- default:
- Y_FAIL();
- };
-}
-
-void TSearchEventsProcessor::HandleWakeup(TEvWakeup::TPtr& ev, const TActorContext& ctx) {
- if (Stopping)
- return;
- switch (ev->Get()->Tag) {
- case StartQueuesListingTag:
- StartQueuesListing(ctx);
- break;
- case QueuesListSessionStartedTag:
- OnQueuesListSessionReady(ctx);
- break;
- case QueuesListPreparedTag:
- OnQueuesListPrepared(ctx);
- break;
- case QueuesListQueryCompleteTag:
- OnQueuesListQueryComplete(ctx);
- break;
- case RunEventsListingTag:
- RunEventsListing(ctx);
- break;
- case EventsListingDoneTag:
- OnEventsListingDone(ctx);
- break;
- case StartCleanupTag:
- StartCleanupSession(ctx);
- break;
- case CleanupSessionReadyTag:
- OnCleanupSessionReady(ctx);
- break;
- case CleanupTxReadyTag:
- OnCleanupTxReady(ctx);
- break;
- case CleanupPreparedTag:
- OnCleanupPrepared(ctx);
- break;
- case CleanupQueryCompleteTag:
- OnCleanupQueryComplete(ctx);
- break;
- case CleanupTxCommittedTag:
- OnCleanupTxCommitted(ctx);
- break;
- case StopAllTag:
- Stopping = true;
- EventsWriter->Close();
- break;
- default:
- Y_FAIL();
- }
-}
-
-void TSearchEventsProcessor::StartQueuesListing(const TActorContext& ctx) {
- LastQueuesKey = {};
- StartSession(QueuesListSessionStartedTag, ctx);
-}
-
-void TSearchEventsProcessor::OnQueuesListSessionReady(const TActorContext &ctx) {
- if (!SessionStarted(ctx)) {
- return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
- }
- PrepareDataQuery(SelectQueuesQuery, QueuesListPreparedTag, ctx);
-}
-
-void TSearchEventsProcessor::OnQueuesListPrepared(const TActorContext &ctx) {
- if (!QueryPrepared(ctx)) {
- return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
- }
- RunQueuesListQuery(ctx, true);
-}
-
-void TSearchEventsProcessor::RunQueuesListQuery(const TActorContext& ctx, bool initial) {
- if (initial) {
- ExistingQueues.clear();
- }
- Y_VERIFY(Session.Defined());
- Y_VERIFY(PreparedQuery.Defined());
-
- auto builder = PreparedQuery.Get()->GetParamsBuilder();
- {
- auto &param = builder.AddParam("$Account");
- param.Utf8(LastQueuesKey.Account);
- param.Build();
- }
- {
- auto &param = builder.AddParam("$QueueName");
- param.Utf8(LastQueuesKey.QueueName);
- param.Build();
- }
- RunPrepared(builder.Build(), QueuesListQueryCompleteTag, ctx);
-}
-
-void TSearchEventsProcessor::OnQueuesListQueryComplete(const TActorContext& ctx) {
- if (!QueryComplete(ctx)) {
- return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
- }
-
- const auto& result = QueryResult.Get()->GetResultSet(0);
-
- TResultSetParser parser(result);
- while (parser.TryNextRow()) {
- TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8();
- TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8();
- TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8();
- ui64 createTs = *parser.ColumnParser("CreatedTimestamp").GetOptionalUint64();
- TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8();
- auto insResult = ExistingQueues.insert(std::make_pair(
- queueName, TQueueEvent{EQueueEventType::Existed, createTs, customName, cloudId, folderId}
- ));
- Y_VERIFY(insResult.second);
- LastQueuesKey.QueueName = queueName;
- LastQueuesKey.Account = cloudId;
- }
- if (result.Truncated()) {
- RunQueuesListQuery(ctx, false);
- } else {
- Send(ctx.SelfID, new TEvWakeup(RunEventsListingTag));
- }
-}
-
-void TSearchEventsProcessor::RunEventsListing(const TActorContext& ctx) {
- QueryResult = Nothing();
- Status = TableClient->RetryOperation<NTable::TDataQueryResult>([this, query = SelectEventsQuery](NTable::TSession session) {
- auto tx = NTable::TTxControl::BeginTx().CommitTx();
- return session.ExecuteDataQuery(
- query, tx, NTable::TExecDataQuerySettings().ClientTimeout(DEFAULT_QUERY_TIMEOUT)
- ).Apply([this](const auto& future) mutable {
- QueryResult = future.GetValue();
- return future;
- });
- });
- Apply(&Status, EventsListingDoneTag, ctx);
-}
-
-void TSearchEventsProcessor::OnEventsListingDone(const TActorContext& ctx) {
- auto& status = Status.GetValue();
- if (!status.IsSuccess()) {
- Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(RunEventsListingTag));
- return;
- }
-
- Y_VERIFY(QueryResult.Defined());
- Y_VERIFY(QueryResult.Get()->GetResultSets().size() == 1);
- const auto& result = QueryResult.Get()->GetResultSet(0);
- TResultSetParser parser(result);
- QueuesEvents.clear();
-
- while (parser.TryNextRow()) {
- TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8();
- TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8();
- ui64 evType = *parser.ColumnParser("EventType").GetOptionalUint64();
- TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8();
- TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8();
- ui64 timestamp = *parser.ColumnParser("EventTimestamp").GetOptionalUint64();
- auto& qEvents = QueuesEvents[queueName];
- auto insResult = qEvents.insert(std::make_pair(
- timestamp, TQueueEvent{EQueueEventType(evType), timestamp, customName, cloudId, folderId}
- ));
- Y_VERIFY(insResult.second);
- }
- ProcessEventsQueue(ctx);
-}
-
-void TSearchEventsProcessor::StartCleanupSession(const TActorContext& ctx) {
- StartSession(CleanupSessionReadyTag, ctx);
-}
-
-void TSearchEventsProcessor::OnCleanupSessionReady(const TActorContext &ctx) {
- if (!SessionStarted(ctx)) {
- return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
- }
- StartTx(CleanupTxReadyTag, ctx);
-}
-
-void TSearchEventsProcessor::OnCleanupTxReady(const TActorContext &ctx) {
- if (!TxStarted(ctx)) {
- return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
- }
- PrepareDataQuery(DeleteEventQuery, CleanupPreparedTag, ctx);
-}
-
-void TSearchEventsProcessor::OnCleanupPrepared(const TActorContext &ctx) {
- if (!QueryPrepared(ctx)) {
- return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
- }
- RunEventsCleanup(ctx);
-}
-
-void TSearchEventsProcessor::RunEventsCleanup(const TActorContext& ctx) {
- Y_VERIFY(PreparedQuery.Defined());
- Y_VERIFY(CurrentTx.Defined());
-
- auto builder = PreparedQuery.Get()->GetParamsBuilder();
- auto &param = builder.AddParam("$Events");
- param.BeginList();
- for (const auto&[qName, events] : QueuesEvents) {
- for (const auto&[_, event]: events) {
- param.AddListItem().BeginStruct().AddMember("Account").Utf8(event.CloudId)
- .AddMember("QueueName").Utf8(qName)
- .AddMember("EventType").Uint64(static_cast<ui64>(event.Type))
- .EndStruct();
-
- }
- }
- param.EndList().Build();
- RunPrepared(builder.Build(), CleanupQueryCompleteTag, ctx);
-}
-
-void TSearchEventsProcessor::OnCleanupQueryComplete(const TActorContext& ctx) {
- if (!QueryComplete(ctx)) {
- return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
- }
- CommitTx(CleanupTxCommittedTag, ctx);
-}
-
-void TSearchEventsProcessor::OnCleanupTxCommitted(const TActorContext &ctx) {
- if (!TxCommitted(ctx)) {
- return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
- }
- QueuesEvents.clear();
- ProcessReindexIfRequired(ctx);
-}
-
-void TSearchEventsProcessor::ProcessEventsQueue(const TActorContext& ctx) {
-
- for (const auto& [qName, events] : QueuesEvents) {
- for (const auto& [ts, event]: events) {
- auto existsIter = ExistingQueues.find(qName);
- if (event.Type == EQueueEventType::Deleted) {
- if (existsIter != ExistingQueues.end() && existsIter->second.CustomName == event.CustomName
- && existsIter->second.Timestamp < ts) {
- ExistingQueues.erase(existsIter);
- }
- } else if (event.Type == EQueueEventType::Created) {
- if (existsIter == ExistingQueues.end()) {
- auto iter = ExistingQueues.insert(std::make_pair(
- qName, event
- )).first;
- iter->second.Type = EQueueEventType::Existed;
- }
- }
- SaveQueueEvent(qName, event, ctx);
- }
- }
- if (!QueuesEvents.empty()) {
- Send(ctx.SelfID, new TEvWakeup(StartCleanupTag));
- } else {
- ProcessReindexIfRequired(ctx);
- }
-}
-
-void TSearchEventsProcessor::ProcessReindexIfRequired(const TActorContext &ctx) {
- if (LastReindex == TInstant::Zero() || LastReindex + ReindexInterval < ctx.Now()) {
- if (ExistingQueues) {
- TStringStream ss;
- NJson::TJsonWriter writer(&ss, false);
- for (const auto&[queue, event] : ExistingQueues) {
- SaveQueueEvent(queue, event, ctx);
- }
- }
- ProcessReindexResult(ctx);
- } else {
- WaitNextCycle(ctx);
- }
-}
-void TSearchEventsProcessor::SendJson(const TString& json, const TActorContext &) {
- EventsWriter->Write(json);
-}
-
-void TSearchEventsProcessor::ProcessReindexResult(const TActorContext &ctx) {
- ReindexComplete.Inc();
- LastReindex = ctx.Now();
- WaitNextCycle(ctx);
-}
-
-void TSearchEventsProcessor::WaitNextCycle(const TActorContext &ctx) {
- ctx.Schedule(RescanInterval, new TEvWakeup(StartQueuesListingTag));
-}
-
-ui64 TSearchEventsProcessor::GetReindexCount() const {
- return ReindexComplete.Val();
-}
-
-NActors::TActorSystem* TSearchEventsProcessor::GetActorSystem() {
- return TActivationContext::ActorSystem();
-}
-
-void TSearchEventsProcessor::StartSession(ui32 evTag, const TActorContext &ctx) {
- Session = Nothing();
- CurrentTx = Nothing();
- PreparedQuery = Nothing();
- QueryResult = Nothing();
- SessionFuture = TableClient->GetSession();
- Apply(&SessionFuture, evTag, ctx);
-}
-
-void TSearchEventsProcessor::PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx) {
- Y_VERIFY(Session.Defined());
- PreparedQuery = Nothing();
- QueryResult = Nothing();
- PrepQueryFuture = Session.Get()->PrepareDataQuery(query);
- Apply(&PrepQueryFuture, evTag, ctx);
-}
-
-void TSearchEventsProcessor::StartTx(ui32 evTag, const TActorContext &ctx) {
- Y_VERIFY(Session.Defined());
- CurrentTx = Nothing();
- TxFuture = Session.Get()->BeginTransaction();
- Apply(&TxFuture, evTag, ctx);
-}
-
-void TSearchEventsProcessor::CommitTx(ui32 evTag, const TActorContext &ctx) {
- Y_VERIFY(Session.Defined());
- Y_VERIFY(CurrentTx.Defined());
- CommitTxFuture = CurrentTx.Get()->Commit();
- Apply(&CommitTxFuture, evTag, ctx);
-}
-
-void TSearchEventsProcessor::RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx) {
- Y_VERIFY(Session.Defined());
- Y_VERIFY(PreparedQuery.Defined());
- QueryResult = Nothing();
- NTable::TTxControl txControl = CurrentTx.Defined() ? NTable::TTxControl::Tx(*CurrentTx)
- : NTable::TTxControl::BeginTx().CommitTx();
- QueryResultFuture = PreparedQuery.Get()->Execute(txControl, std::forward<NYdb::TParams>(params));
- Apply(&QueryResultFuture, evTag, ctx);
-}
-
-bool TSearchEventsProcessor::SessionStarted(const TActorContext&) {
- Y_VERIFY(SessionFuture.HasValue());
- auto& value = SessionFuture.GetValueSync();
- if (!value.IsSuccess())
- return false;
- Session = value.GetSession();
- return true;
-}
-
-bool TSearchEventsProcessor::TxStarted(const TActorContext&) {
- Y_VERIFY(Session.Defined());
- Y_VERIFY(TxFuture.HasValue());
- auto& value = TxFuture.GetValueSync();
- if (!value.IsSuccess())
- return false;
- CurrentTx = value.GetTransaction();
- return true;
-}
-bool TSearchEventsProcessor::TxCommitted(const TActorContext&) {
- Y_VERIFY(Session.Defined());
- Y_VERIFY(CurrentTx.Defined());
- Y_VERIFY(CommitTxFuture.HasValue());
- auto& value = CommitTxFuture.GetValueSync();
- if (!value.IsSuccess())
- return false;
- CurrentTx = Nothing();
- return true;
-}
-
-bool TSearchEventsProcessor::QueryPrepared(const TActorContext&) {
- Y_VERIFY(Session.Defined());
- Y_VERIFY(PrepQueryFuture.HasValue());
- auto& value = PrepQueryFuture.GetValueSync();
- if (!value.IsSuccess()) {
- return false;
- }
- PreparedQuery = value.GetQuery();
- return true;
-}
-bool TSearchEventsProcessor::QueryComplete(const TActorContext&) {
- Y_VERIFY(Session.Defined());
- Y_VERIFY(QueryResultFuture.HasValue());
- QueryResult = QueryResultFuture.GetValueSync();
- if (!QueryResult.Get()->IsSuccess()) {
- return false;
- }
- return true;
-}
-
-
-void TSearchEventsProcessor::SaveQueueEvent(
-
- const TString& queueName, const TQueueEvent& event, const TActorContext& ctx
-) {
- auto tsIsoString = TInstant::MilliSeconds(event.Timestamp).ToIsoStringLocal();
- auto nowIsoString = TInstant::Now().ToIsoStringLocal();
- TStringStream ss;
- NJson::TJsonWriter writer(&ss, false);
-
- writer.OpenMap();
- {
- writer.Write("resource_type", "message-queue");
- writer.Write("timestamp", tsIsoString);
- writer.Write("resource_id", queueName);
- writer.Write("name", event.CustomName);
- writer.Write("service", "message-queue");
- if (event.Type == EQueueEventType::Deleted)
- writer.Write("deleted", tsIsoString);
- if (event.Type == EQueueEventType::Existed)
- writer.Write("reindex_timestamp", nowIsoString);
- writer.Write("permission", "ymq.queues.list");
- writer.Write("cloud_id", event.CloudId);
- writer.Write("folder_id", event.FolderId);
- writer.OpenArray("resource_path");
- writer.OpenMap();
- writer.Write("resource_type", "resource-manager.folder");
- writer.Write("resource_id", event.FolderId);
- writer.CloseMap();
- writer.CloseArray(); // resource_path
- }
- writer.CloseMap();
- writer.Flush();
- SendJson(ss.Str(), ctx);
-}
-
-void IEventsWriterWrapper::Close() {
- if (!Closed) {
- Closed = true;
- CloseImpl();
- }
-}
-
+
+namespace NKikimr::NSQS {
+using namespace NActors;
+using namespace NJson;
+using namespace NYdb;
+
+constexpr TDuration DEFAULT_RETRY_TIMEOUT = TDuration::Seconds(10);
+constexpr TDuration SHORT_RETRY_TIMEOUT = TDuration::Seconds(2);
+constexpr TDuration DEFAULT_QUERY_TIMEOUT = TDuration::Seconds(30);
+
+TSearchEventsProcessor::TSearchEventsProcessor(
+ const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval,
+ const TSimpleSharedPtr<NTable::TTableClient>& tableClient,
+ IEventsWriterWrapper::TPtr eventsWriter,
+ bool waitForWake
+)
+ : RescanInterval(rescanInterval)
+ , ReindexInterval(reindexInterval)
+ , TableClient(tableClient)
+ , EventsWriter(eventsWriter)
+ , WaitForWake(waitForWake)
+{
+ InitQueries(root);
+}
+
+void TSearchEventsProcessor::InitQueries(const TString& root) {
+ auto getTableFullPath = [&](const TString& tableName) {
+ if (!root.empty()) {
+ return root + "/" + tableName;
+ } else {
+ return tableName;
+ }
+ };
+ TString eventsTablePath = getTableFullPath(".Events");
+
+ SelectQueuesQuery = TStringBuilder()
+ << "--!syntax_v1\n"
+ << "DECLARE $Account as Utf8; DECLARE $QueueName as Utf8; "
+ << "SELECT Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId from `"
+ << getTableFullPath(".Queues") << "` "
+ << "WHERE Account > $Account OR (Account = $Account AND QueueName > $QueueName);";
+
+ SelectEventsQuery = TStringBuilder()
+ << "--!syntax_v1\n"
+ << "SELECT Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId "
+ << "FROM `"<< eventsTablePath << "`;";
+
+ DeleteEventQuery = TStringBuilder()
+ << "--!syntax_v1\n"
+ << "DECLARE $Events AS List<Struct<Account:Utf8, QueueName:Utf8, EventType:Uint64>>;\n"
+ << "$EventsSource = (SELECT item.Account AS Account, item.QueueName AS QueueName, item.EventType AS EventType \n"
+ << "FROM (SELECT $Events AS events) FLATTEN LIST BY events as item);\n"
+ << "DELETE FROM `" << eventsTablePath << "` ON SELECT * FROM $EventsSource;";
+}
+
+TSearchEventsProcessor::~TSearchEventsProcessor() {
+ EventsWriter->Close();
+}
+
+void TSearchEventsProcessor::Bootstrap(const TActorContext& ctx) {
+ Become(&TSearchEventsProcessor::StateFunc);
+ if (!WaitForWake)
+ ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
+}
+
+STFUNC(TSearchEventsProcessor::StateFunc) {
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvWakeup, HandleWakeup);
+ default:
+ Y_FAIL();
+ };
+}
+
+void TSearchEventsProcessor::HandleWakeup(TEvWakeup::TPtr& ev, const TActorContext& ctx) {
+ if (Stopping)
+ return;
+ switch (ev->Get()->Tag) {
+ case StartQueuesListingTag:
+ StartQueuesListing(ctx);
+ break;
+ case QueuesListSessionStartedTag:
+ OnQueuesListSessionReady(ctx);
+ break;
+ case QueuesListPreparedTag:
+ OnQueuesListPrepared(ctx);
+ break;
+ case QueuesListQueryCompleteTag:
+ OnQueuesListQueryComplete(ctx);
+ break;
+ case RunEventsListingTag:
+ RunEventsListing(ctx);
+ break;
+ case EventsListingDoneTag:
+ OnEventsListingDone(ctx);
+ break;
+ case StartCleanupTag:
+ StartCleanupSession(ctx);
+ break;
+ case CleanupSessionReadyTag:
+ OnCleanupSessionReady(ctx);
+ break;
+ case CleanupTxReadyTag:
+ OnCleanupTxReady(ctx);
+ break;
+ case CleanupPreparedTag:
+ OnCleanupPrepared(ctx);
+ break;
+ case CleanupQueryCompleteTag:
+ OnCleanupQueryComplete(ctx);
+ break;
+ case CleanupTxCommittedTag:
+ OnCleanupTxCommitted(ctx);
+ break;
+ case StopAllTag:
+ Stopping = true;
+ EventsWriter->Close();
+ break;
+ default:
+ Y_FAIL();
+ }
+}
+
+void TSearchEventsProcessor::StartQueuesListing(const TActorContext& ctx) {
+ LastQueuesKey = {};
+ StartSession(QueuesListSessionStartedTag, ctx);
+}
+
+void TSearchEventsProcessor::OnQueuesListSessionReady(const TActorContext &ctx) {
+ if (!SessionStarted(ctx)) {
+ return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
+ }
+ PrepareDataQuery(SelectQueuesQuery, QueuesListPreparedTag, ctx);
+}
+
+void TSearchEventsProcessor::OnQueuesListPrepared(const TActorContext &ctx) {
+ if (!QueryPrepared(ctx)) {
+ return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
+ }
+ RunQueuesListQuery(ctx, true);
+}
+
+void TSearchEventsProcessor::RunQueuesListQuery(const TActorContext& ctx, bool initial) {
+ if (initial) {
+ ExistingQueues.clear();
+ }
+ Y_VERIFY(Session.Defined());
+ Y_VERIFY(PreparedQuery.Defined());
+
+ auto builder = PreparedQuery.Get()->GetParamsBuilder();
+ {
+ auto &param = builder.AddParam("$Account");
+ param.Utf8(LastQueuesKey.Account);
+ param.Build();
+ }
+ {
+ auto &param = builder.AddParam("$QueueName");
+ param.Utf8(LastQueuesKey.QueueName);
+ param.Build();
+ }
+ RunPrepared(builder.Build(), QueuesListQueryCompleteTag, ctx);
+}
+
+void TSearchEventsProcessor::OnQueuesListQueryComplete(const TActorContext& ctx) {
+ if (!QueryComplete(ctx)) {
+ return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartQueuesListingTag));
+ }
+
+ const auto& result = QueryResult.Get()->GetResultSet(0);
+
+ TResultSetParser parser(result);
+ while (parser.TryNextRow()) {
+ TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8();
+ TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8();
+ TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8();
+ ui64 createTs = *parser.ColumnParser("CreatedTimestamp").GetOptionalUint64();
+ TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8();
+ auto insResult = ExistingQueues.insert(std::make_pair(
+ queueName, TQueueEvent{EQueueEventType::Existed, createTs, customName, cloudId, folderId}
+ ));
+ Y_VERIFY(insResult.second);
+ LastQueuesKey.QueueName = queueName;
+ LastQueuesKey.Account = cloudId;
+ }
+ if (result.Truncated()) {
+ RunQueuesListQuery(ctx, false);
+ } else {
+ Send(ctx.SelfID, new TEvWakeup(RunEventsListingTag));
+ }
+}
+
+void TSearchEventsProcessor::RunEventsListing(const TActorContext& ctx) {
+ QueryResult = Nothing();
+ Status = TableClient->RetryOperation<NTable::TDataQueryResult>([this, query = SelectEventsQuery](NTable::TSession session) {
+ auto tx = NTable::TTxControl::BeginTx().CommitTx();
+ return session.ExecuteDataQuery(
+ query, tx, NTable::TExecDataQuerySettings().ClientTimeout(DEFAULT_QUERY_TIMEOUT)
+ ).Apply([this](const auto& future) mutable {
+ QueryResult = future.GetValue();
+ return future;
+ });
+ });
+ Apply(&Status, EventsListingDoneTag, ctx);
+}
+
+void TSearchEventsProcessor::OnEventsListingDone(const TActorContext& ctx) {
+ auto& status = Status.GetValue();
+ if (!status.IsSuccess()) {
+ Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(RunEventsListingTag));
+ return;
+ }
+
+ Y_VERIFY(QueryResult.Defined());
+ Y_VERIFY(QueryResult.Get()->GetResultSets().size() == 1);
+ const auto& result = QueryResult.Get()->GetResultSet(0);
+ TResultSetParser parser(result);
+ QueuesEvents.clear();
+
+ while (parser.TryNextRow()) {
+ TString cloudId = *parser.ColumnParser("Account").GetOptionalUtf8();
+ TString queueName = *parser.ColumnParser("QueueName").GetOptionalUtf8();
+ ui64 evType = *parser.ColumnParser("EventType").GetOptionalUint64();
+ TString customName = *parser.ColumnParser("CustomQueueName").GetOptionalUtf8();
+ TString folderId = *parser.ColumnParser("FolderId").GetOptionalUtf8();
+ ui64 timestamp = *parser.ColumnParser("EventTimestamp").GetOptionalUint64();
+ auto& qEvents = QueuesEvents[queueName];
+ auto insResult = qEvents.insert(std::make_pair(
+ timestamp, TQueueEvent{EQueueEventType(evType), timestamp, customName, cloudId, folderId}
+ ));
+ Y_VERIFY(insResult.second);
+ }
+ ProcessEventsQueue(ctx);
+}
+
+void TSearchEventsProcessor::StartCleanupSession(const TActorContext& ctx) {
+ StartSession(CleanupSessionReadyTag, ctx);
+}
+
+void TSearchEventsProcessor::OnCleanupSessionReady(const TActorContext &ctx) {
+ if (!SessionStarted(ctx)) {
+ return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
+ }
+ StartTx(CleanupTxReadyTag, ctx);
+}
+
+void TSearchEventsProcessor::OnCleanupTxReady(const TActorContext &ctx) {
+ if (!TxStarted(ctx)) {
+ return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
+ }
+ PrepareDataQuery(DeleteEventQuery, CleanupPreparedTag, ctx);
+}
+
+void TSearchEventsProcessor::OnCleanupPrepared(const TActorContext &ctx) {
+ if (!QueryPrepared(ctx)) {
+ return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
+ }
+ RunEventsCleanup(ctx);
+}
+
+void TSearchEventsProcessor::RunEventsCleanup(const TActorContext& ctx) {
+ Y_VERIFY(PreparedQuery.Defined());
+ Y_VERIFY(CurrentTx.Defined());
+
+ auto builder = PreparedQuery.Get()->GetParamsBuilder();
+ auto &param = builder.AddParam("$Events");
+ param.BeginList();
+ for (const auto&[qName, events] : QueuesEvents) {
+ for (const auto&[_, event]: events) {
+ param.AddListItem().BeginStruct().AddMember("Account").Utf8(event.CloudId)
+ .AddMember("QueueName").Utf8(qName)
+ .AddMember("EventType").Uint64(static_cast<ui64>(event.Type))
+ .EndStruct();
+
+ }
+ }
+ param.EndList().Build();
+ RunPrepared(builder.Build(), CleanupQueryCompleteTag, ctx);
+}
+
+void TSearchEventsProcessor::OnCleanupQueryComplete(const TActorContext& ctx) {
+ if (!QueryComplete(ctx)) {
+ return ctx.Schedule(SHORT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
+ }
+ CommitTx(CleanupTxCommittedTag, ctx);
+}
+
+void TSearchEventsProcessor::OnCleanupTxCommitted(const TActorContext &ctx) {
+ if (!TxCommitted(ctx)) {
+ return ctx.Schedule(DEFAULT_RETRY_TIMEOUT, new TEvWakeup(StartCleanupTag));
+ }
+ QueuesEvents.clear();
+ ProcessReindexIfRequired(ctx);
+}
+
+void TSearchEventsProcessor::ProcessEventsQueue(const TActorContext& ctx) {
+
+ for (const auto& [qName, events] : QueuesEvents) {
+ for (const auto& [ts, event]: events) {
+ auto existsIter = ExistingQueues.find(qName);
+ if (event.Type == EQueueEventType::Deleted) {
+ if (existsIter != ExistingQueues.end() && existsIter->second.CustomName == event.CustomName
+ && existsIter->second.Timestamp < ts) {
+ ExistingQueues.erase(existsIter);
+ }
+ } else if (event.Type == EQueueEventType::Created) {
+ if (existsIter == ExistingQueues.end()) {
+ auto iter = ExistingQueues.insert(std::make_pair(
+ qName, event
+ )).first;
+ iter->second.Type = EQueueEventType::Existed;
+ }
+ }
+ SaveQueueEvent(qName, event, ctx);
+ }
+ }
+ if (!QueuesEvents.empty()) {
+ Send(ctx.SelfID, new TEvWakeup(StartCleanupTag));
+ } else {
+ ProcessReindexIfRequired(ctx);
+ }
+}
+
+void TSearchEventsProcessor::ProcessReindexIfRequired(const TActorContext &ctx) {
+ if (LastReindex == TInstant::Zero() || LastReindex + ReindexInterval < ctx.Now()) {
+ if (ExistingQueues) {
+ TStringStream ss;
+ NJson::TJsonWriter writer(&ss, false);
+ for (const auto&[queue, event] : ExistingQueues) {
+ SaveQueueEvent(queue, event, ctx);
+ }
+ }
+ ProcessReindexResult(ctx);
+ } else {
+ WaitNextCycle(ctx);
+ }
+}
+void TSearchEventsProcessor::SendJson(const TString& json, const TActorContext &) {
+ EventsWriter->Write(json);
+}
+
+void TSearchEventsProcessor::ProcessReindexResult(const TActorContext &ctx) {
+ ReindexComplete.Inc();
+ LastReindex = ctx.Now();
+ WaitNextCycle(ctx);
+}
+
+void TSearchEventsProcessor::WaitNextCycle(const TActorContext &ctx) {
+ ctx.Schedule(RescanInterval, new TEvWakeup(StartQueuesListingTag));
+}
+
+ui64 TSearchEventsProcessor::GetReindexCount() const {
+ return ReindexComplete.Val();
+}
+
+NActors::TActorSystem* TSearchEventsProcessor::GetActorSystem() {
+ return TActivationContext::ActorSystem();
+}
+
+void TSearchEventsProcessor::StartSession(ui32 evTag, const TActorContext &ctx) {
+ Session = Nothing();
+ CurrentTx = Nothing();
+ PreparedQuery = Nothing();
+ QueryResult = Nothing();
+ SessionFuture = TableClient->GetSession();
+ Apply(&SessionFuture, evTag, ctx);
+}
+
+void TSearchEventsProcessor::PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx) {
+ Y_VERIFY(Session.Defined());
+ PreparedQuery = Nothing();
+ QueryResult = Nothing();
+ PrepQueryFuture = Session.Get()->PrepareDataQuery(query);
+ Apply(&PrepQueryFuture, evTag, ctx);
+}
+
+void TSearchEventsProcessor::StartTx(ui32 evTag, const TActorContext &ctx) {
+ Y_VERIFY(Session.Defined());
+ CurrentTx = Nothing();
+ TxFuture = Session.Get()->BeginTransaction();
+ Apply(&TxFuture, evTag, ctx);
+}
+
+void TSearchEventsProcessor::CommitTx(ui32 evTag, const TActorContext &ctx) {
+ Y_VERIFY(Session.Defined());
+ Y_VERIFY(CurrentTx.Defined());
+ CommitTxFuture = CurrentTx.Get()->Commit();
+ Apply(&CommitTxFuture, evTag, ctx);
+}
+
+void TSearchEventsProcessor::RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx) {
+ Y_VERIFY(Session.Defined());
+ Y_VERIFY(PreparedQuery.Defined());
+ QueryResult = Nothing();
+ NTable::TTxControl txControl = CurrentTx.Defined() ? NTable::TTxControl::Tx(*CurrentTx)
+ : NTable::TTxControl::BeginTx().CommitTx();
+ QueryResultFuture = PreparedQuery.Get()->Execute(txControl, std::forward<NYdb::TParams>(params));
+ Apply(&QueryResultFuture, evTag, ctx);
+}
+
+bool TSearchEventsProcessor::SessionStarted(const TActorContext&) {
+ Y_VERIFY(SessionFuture.HasValue());
+ auto& value = SessionFuture.GetValueSync();
+ if (!value.IsSuccess())
+ return false;
+ Session = value.GetSession();
+ return true;
+}
+
+bool TSearchEventsProcessor::TxStarted(const TActorContext&) {
+ Y_VERIFY(Session.Defined());
+ Y_VERIFY(TxFuture.HasValue());
+ auto& value = TxFuture.GetValueSync();
+ if (!value.IsSuccess())
+ return false;
+ CurrentTx = value.GetTransaction();
+ return true;
+}
+bool TSearchEventsProcessor::TxCommitted(const TActorContext&) {
+ Y_VERIFY(Session.Defined());
+ Y_VERIFY(CurrentTx.Defined());
+ Y_VERIFY(CommitTxFuture.HasValue());
+ auto& value = CommitTxFuture.GetValueSync();
+ if (!value.IsSuccess())
+ return false;
+ CurrentTx = Nothing();
+ return true;
+}
+
+bool TSearchEventsProcessor::QueryPrepared(const TActorContext&) {
+ Y_VERIFY(Session.Defined());
+ Y_VERIFY(PrepQueryFuture.HasValue());
+ auto& value = PrepQueryFuture.GetValueSync();
+ if (!value.IsSuccess()) {
+ return false;
+ }
+ PreparedQuery = value.GetQuery();
+ return true;
+}
+bool TSearchEventsProcessor::QueryComplete(const TActorContext&) {
+ Y_VERIFY(Session.Defined());
+ Y_VERIFY(QueryResultFuture.HasValue());
+ QueryResult = QueryResultFuture.GetValueSync();
+ if (!QueryResult.Get()->IsSuccess()) {
+ return false;
+ }
+ return true;
+}
+
+
+void TSearchEventsProcessor::SaveQueueEvent(
+
+ const TString& queueName, const TQueueEvent& event, const TActorContext& ctx
+) {
+ auto tsIsoString = TInstant::MilliSeconds(event.Timestamp).ToIsoStringLocal();
+ auto nowIsoString = TInstant::Now().ToIsoStringLocal();
+ TStringStream ss;
+ NJson::TJsonWriter writer(&ss, false);
+
+ writer.OpenMap();
+ {
+ writer.Write("resource_type", "message-queue");
+ writer.Write("timestamp", tsIsoString);
+ writer.Write("resource_id", queueName);
+ writer.Write("name", event.CustomName);
+ writer.Write("service", "message-queue");
+ if (event.Type == EQueueEventType::Deleted)
+ writer.Write("deleted", tsIsoString);
+ if (event.Type == EQueueEventType::Existed)
+ writer.Write("reindex_timestamp", nowIsoString);
+ writer.Write("permission", "ymq.queues.list");
+ writer.Write("cloud_id", event.CloudId);
+ writer.Write("folder_id", event.FolderId);
+ writer.OpenArray("resource_path");
+ writer.OpenMap();
+ writer.Write("resource_type", "resource-manager.folder");
+ writer.Write("resource_id", event.FolderId);
+ writer.CloseMap();
+ writer.CloseArray(); // resource_path
+ }
+ writer.CloseMap();
+ writer.Flush();
+ SendJson(ss.Str(), ctx);
+}
+
+void IEventsWriterWrapper::Close() {
+ if (!Closed) {
+ Closed = true;
+ CloseImpl();
+ }
+}
+
}; // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/index_events_processor.h b/ydb/core/ymq/actor/index_events_processor.h
index 3efde48f04..a8ae462064 100644
--- a/ydb/core/ymq/actor/index_events_processor.h
+++ b/ydb/core/ymq/actor/index_events_processor.h
@@ -1,148 +1,148 @@
-#pragma once
-
-#include "events.h"
+#pragma once
+
+#include "events.h"
#include <ydb/core/ymq/base/events_writer_iface.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
-#include <library/cpp/actors/core/actor_bootstrapped.h>
-#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/actors/core/actorsystem.h>
-#include <library/cpp/json/json_writer.h>
-#include <util/stream/file.h>
-
-
-
+#include <library/cpp/actors/core/actor_bootstrapped.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/actorsystem.h>
+#include <library/cpp/json/json_writer.h>
+#include <util/stream/file.h>
+
+
+
namespace NKikimr::NSQS {
-
-
-
-class TSearchEventsProcessor : public NActors::TActorBootstrapped<TSearchEventsProcessor> {
-using TActorContext = NActors::TActorContext;
-friend class TIndexProcesorTests;
-public:
-
- TSearchEventsProcessor(const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval,
- const TSimpleSharedPtr<NYdb::NTable::TTableClient>& tableClient,
- IEventsWriterWrapper::TPtr eventsWriter,
- bool waitForWake = false);
-
- ~TSearchEventsProcessor();
- void Bootstrap(const TActorContext& ctx);
- ui64 GetReindexCount() const;
-
-private:
- TString SelectQueuesQuery;
- TString SelectEventsQuery;
- TString DeleteEventQuery;
- TDuration RescanInterval;
- TDuration ReindexInterval;
- TInstant LastReindex = TInstant::Zero();
- TSimpleSharedPtr<NYdb::NTable::TTableClient> TableClient;
- IEventsWriterWrapper::TPtr EventsWriter;
-
- NYdb::TAsyncStatus Status;
- NYdb::NTable::TAsyncCreateSessionResult SessionFuture;
- NYdb::NTable::TAsyncBeginTransactionResult TxFuture;
- NYdb::NTable::TAsyncCommitTransactionResult CommitTxFuture;
- NYdb::NTable::TAsyncPrepareQueryResult PrepQueryFuture;
- NYdb::NTable::TAsyncDataQueryResult QueryResultFuture;
-
- TMaybe<NYdb::NTable::TSession> Session;
- TMaybe<NYdb::NTable::TDataQuery> PreparedQuery;
- TMaybe<NYdb::NTable::TTransaction> CurrentTx;
- TMaybe<NYdb::NTable::TDataQueryResult> QueryResult;
-
- bool WaitForWake = false;
- bool Stopping = false;
- enum class EQueueEventType {
- Deleted = 0,
- Created = 1,
- Existed = 2,
- };
- struct TQueueEvent {
- EQueueEventType Type;
- ui64 Timestamp;
- TString CustomName;
- TString CloudId;
- TString FolderId;
- };
- struct TQueueTableKey {
- TString Account;
- TString QueueName;
- };
- TQueueTableKey LastQueuesKey = {};
-
- THashMap<TString, TMap<ui64, TQueueEvent>> QueuesEvents;
- THashMap<TString, TQueueEvent> ExistingQueues;
-
- TAtomicCounter ReindexComplete = 0;
-
- STATEFN(StateFunc);
- void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx);
-
- void StartQueuesListing(const TActorContext& ctx);
- void OnQueuesListSessionReady(const TActorContext& ctx);
- void OnQueuesListPrepared(const TActorContext& ctx);
- void RunQueuesListQuery(const TActorContext& ctx, bool initial = false);
- void OnQueuesListQueryComplete(const TActorContext& ctx);
-
- void RunEventsListing(const TActorContext& ctx);
- void OnEventsListingDone(const TActorContext& ctx);
- void StartCleanupSession(const TActorContext& ctx);
- void OnCleanupSessionReady(const TActorContext& ctx);
- void OnCleanupTxReady(const TActorContext& ctx);
- void OnCleanupPrepared(const TActorContext& ctx);
- void RunEventsCleanup(const TActorContext& ctx);
- void OnCleanupQueryComplete(const TActorContext&ctx);
- void OnCleanupTxCommitted(const TActorContext&ctx);
-
- void ProcessEventsQueue(const TActorContext& ctx);
- void ProcessReindexIfRequired(const TActorContext& ctx);
- void ProcessReindexResult(const TActorContext& ctx);
- void WaitNextCycle(const TActorContext& ctx);
-
-
- void StartSession(ui32 evTag, const TActorContext& ctx);
- void PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx);
- void StartTx(ui32 evTag, const TActorContext& ctx);
- void CommitTx(ui32 evTag, const TActorContext& ctx);
- void RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx);
- bool SessionStarted(const TActorContext& ctx);
- bool QueryPrepared(const TActorContext& ctx);
- bool QueryComplete(const TActorContext& ctx);
- bool TxStarted(const TActorContext& ctx);
- bool TxCommitted(const TActorContext& ctx);
-
-
- template<class TFutureType>
- void Apply(TFutureType* future, ui32 evTag, const TActorContext& ctx) {
- future->Subscribe(
- [evTag, as = GetActorSystem(), selfId = ctx.SelfID](const auto&)
- {as->Send(selfId, new TEvWakeup(evTag));}
- );
- }
- void InitQueries(const TString& root);
- void SaveQueueEvent(const TString& queueName, const TQueueEvent& event, const TActorContext& ctx);
- void SendJson(const TString& json, const TActorContext &ctx);
- static NActors::TActorSystem* GetActorSystem();
-
-
- constexpr static ui32 StartQueuesListingTag = 1;
- constexpr static ui32 QueuesListSessionStartedTag = 2;
- constexpr static ui32 QueuesListPreparedTag = 3;
- constexpr static ui32 QueuesListQueryCompleteTag = 4;
-
- constexpr static ui32 RunEventsListingTag = 10;
- constexpr static ui32 EventsListingDoneTag = 11;
-
- constexpr static ui32 StartCleanupTag = 20;
- constexpr static ui32 CleanupSessionReadyTag = 21;
- constexpr static ui32 CleanupTxReadyTag = 22;
- constexpr static ui32 CleanupPreparedTag = 23;
- constexpr static ui32 CleanupQueryCompleteTag = 24;
- constexpr static ui32 CleanupTxCommittedTag = 25;
-
- constexpr static ui32 StopAllTag = 99;
-
-
-};
+
+
+
+class TSearchEventsProcessor : public NActors::TActorBootstrapped<TSearchEventsProcessor> {
+using TActorContext = NActors::TActorContext;
+friend class TIndexProcesorTests;
+public:
+
+ TSearchEventsProcessor(const TString& root, const TDuration& reindexInterval, const TDuration& rescanInterval,
+ const TSimpleSharedPtr<NYdb::NTable::TTableClient>& tableClient,
+ IEventsWriterWrapper::TPtr eventsWriter,
+ bool waitForWake = false);
+
+ ~TSearchEventsProcessor();
+ void Bootstrap(const TActorContext& ctx);
+ ui64 GetReindexCount() const;
+
+private:
+ TString SelectQueuesQuery;
+ TString SelectEventsQuery;
+ TString DeleteEventQuery;
+ TDuration RescanInterval;
+ TDuration ReindexInterval;
+ TInstant LastReindex = TInstant::Zero();
+ TSimpleSharedPtr<NYdb::NTable::TTableClient> TableClient;
+ IEventsWriterWrapper::TPtr EventsWriter;
+
+ NYdb::TAsyncStatus Status;
+ NYdb::NTable::TAsyncCreateSessionResult SessionFuture;
+ NYdb::NTable::TAsyncBeginTransactionResult TxFuture;
+ NYdb::NTable::TAsyncCommitTransactionResult CommitTxFuture;
+ NYdb::NTable::TAsyncPrepareQueryResult PrepQueryFuture;
+ NYdb::NTable::TAsyncDataQueryResult QueryResultFuture;
+
+ TMaybe<NYdb::NTable::TSession> Session;
+ TMaybe<NYdb::NTable::TDataQuery> PreparedQuery;
+ TMaybe<NYdb::NTable::TTransaction> CurrentTx;
+ TMaybe<NYdb::NTable::TDataQueryResult> QueryResult;
+
+ bool WaitForWake = false;
+ bool Stopping = false;
+ enum class EQueueEventType {
+ Deleted = 0,
+ Created = 1,
+ Existed = 2,
+ };
+ struct TQueueEvent {
+ EQueueEventType Type;
+ ui64 Timestamp;
+ TString CustomName;
+ TString CloudId;
+ TString FolderId;
+ };
+ struct TQueueTableKey {
+ TString Account;
+ TString QueueName;
+ };
+ TQueueTableKey LastQueuesKey = {};
+
+ THashMap<TString, TMap<ui64, TQueueEvent>> QueuesEvents;
+ THashMap<TString, TQueueEvent> ExistingQueues;
+
+ TAtomicCounter ReindexComplete = 0;
+
+ STATEFN(StateFunc);
+ void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx);
+
+ void StartQueuesListing(const TActorContext& ctx);
+ void OnQueuesListSessionReady(const TActorContext& ctx);
+ void OnQueuesListPrepared(const TActorContext& ctx);
+ void RunQueuesListQuery(const TActorContext& ctx, bool initial = false);
+ void OnQueuesListQueryComplete(const TActorContext& ctx);
+
+ void RunEventsListing(const TActorContext& ctx);
+ void OnEventsListingDone(const TActorContext& ctx);
+ void StartCleanupSession(const TActorContext& ctx);
+ void OnCleanupSessionReady(const TActorContext& ctx);
+ void OnCleanupTxReady(const TActorContext& ctx);
+ void OnCleanupPrepared(const TActorContext& ctx);
+ void RunEventsCleanup(const TActorContext& ctx);
+ void OnCleanupQueryComplete(const TActorContext&ctx);
+ void OnCleanupTxCommitted(const TActorContext&ctx);
+
+ void ProcessEventsQueue(const TActorContext& ctx);
+ void ProcessReindexIfRequired(const TActorContext& ctx);
+ void ProcessReindexResult(const TActorContext& ctx);
+ void WaitNextCycle(const TActorContext& ctx);
+
+
+ void StartSession(ui32 evTag, const TActorContext& ctx);
+ void PrepareDataQuery(const TString& query, ui32 evTag, const TActorContext& ctx);
+ void StartTx(ui32 evTag, const TActorContext& ctx);
+ void CommitTx(ui32 evTag, const TActorContext& ctx);
+ void RunPrepared(NYdb::TParams&& params, ui32 evTag, const TActorContext& ctx);
+ bool SessionStarted(const TActorContext& ctx);
+ bool QueryPrepared(const TActorContext& ctx);
+ bool QueryComplete(const TActorContext& ctx);
+ bool TxStarted(const TActorContext& ctx);
+ bool TxCommitted(const TActorContext& ctx);
+
+
+ template<class TFutureType>
+ void Apply(TFutureType* future, ui32 evTag, const TActorContext& ctx) {
+ future->Subscribe(
+ [evTag, as = GetActorSystem(), selfId = ctx.SelfID](const auto&)
+ {as->Send(selfId, new TEvWakeup(evTag));}
+ );
+ }
+ void InitQueries(const TString& root);
+ void SaveQueueEvent(const TString& queueName, const TQueueEvent& event, const TActorContext& ctx);
+ void SendJson(const TString& json, const TActorContext &ctx);
+ static NActors::TActorSystem* GetActorSystem();
+
+
+ constexpr static ui32 StartQueuesListingTag = 1;
+ constexpr static ui32 QueuesListSessionStartedTag = 2;
+ constexpr static ui32 QueuesListPreparedTag = 3;
+ constexpr static ui32 QueuesListQueryCompleteTag = 4;
+
+ constexpr static ui32 RunEventsListingTag = 10;
+ constexpr static ui32 EventsListingDoneTag = 11;
+
+ constexpr static ui32 StartCleanupTag = 20;
+ constexpr static ui32 CleanupSessionReadyTag = 21;
+ constexpr static ui32 CleanupTxReadyTag = 22;
+ constexpr static ui32 CleanupPreparedTag = 23;
+ constexpr static ui32 CleanupQueryCompleteTag = 24;
+ constexpr static ui32 CleanupTxCommittedTag = 25;
+
+ constexpr static ui32 StopAllTag = 99;
+
+
+};
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/proxy_actor.cpp b/ydb/core/ymq/actor/proxy_actor.cpp
index 426977cb11..98d85210a5 100644
--- a/ydb/core/ymq/actor/proxy_actor.cpp
+++ b/ydb/core/ymq/actor/proxy_actor.cpp
@@ -145,19 +145,19 @@ void TProxyActor::HandleWakeup(TEvWakeup::TPtr&) {
RLOG_SQS_ERROR("Proxy request timeout. User [" << UserName_ << "] Queue [" << QueueName_ << "] Action [" << actionName << "]");
if (QueueCounters_) {
- INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second);
+ INC_COUNTER_COUPLE(QueueCounters_, RequestTimeouts, request_timeouts_count_per_second);
} else {
- auto rootCounters = TIntrusivePtrCntrCouple{
- GetSqsServiceCounters(AppData()->Counters, "core"),
- GetYmqPublicCounters(AppData()->Counters)
- };
- auto [userCountersCouple, queueCountersCouple] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, QueueName_));
- if (queueCountersCouple.SqsCounters) {
- queueCountersCouple.SqsCounters->GetCounter("RequestTimeouts", true)->Inc();
+ auto rootCounters = TIntrusivePtrCntrCouple{
+ GetSqsServiceCounters(AppData()->Counters, "core"),
+ GetYmqPublicCounters(AppData()->Counters)
+ };
+ auto [userCountersCouple, queueCountersCouple] = GetUserAndQueueCounters(rootCounters, TQueuePath(Cfg().GetRoot(), UserName_, QueueName_));
+ if (queueCountersCouple.SqsCounters) {
+ queueCountersCouple.SqsCounters->GetCounter("RequestTimeouts", true)->Inc();
+ }
+ if (queueCountersCouple.YmqCounters) {
+ queueCountersCouple.YmqCounters->GetCounter("request_timeouts_count_per_second", true)->Inc();
}
- if (queueCountersCouple.YmqCounters) {
- queueCountersCouple.YmqCounters->GetCounter("request_timeouts_count_per_second", true)->Inc();
- }
}
SendErrorAndDie(NErrors::TIMEOUT);
diff --git a/ydb/core/ymq/actor/proxy_service.cpp b/ydb/core/ymq/actor/proxy_service.cpp
index f7aeb601a6..1eecd85103 100644
--- a/ydb/core/ymq/actor/proxy_service.cpp
+++ b/ydb/core/ymq/actor/proxy_service.cpp
@@ -60,8 +60,8 @@ void TSqsProxyService::Bootstrap() {
LOG_SQS_INFO("Start SQS proxy service actor");
Become(&TThis::StateFunc);
- SqsCounters_ = GetSqsServiceCounters(AppData()->Counters, "core");
- YmqPublicCounters_ = GetYmqPublicCounters(AppData()->Counters);
+ SqsCounters_ = GetSqsServiceCounters(AppData()->Counters, "core");
+ YmqPublicCounters_ = GetYmqPublicCounters(AppData()->Counters);
}
void TSqsProxyService::HandleExecuted(TSqsEvents::TEvExecuted::TPtr& ev) {
diff --git a/ydb/core/ymq/actor/proxy_service.h b/ydb/core/ymq/actor/proxy_service.h
index c03db657fa..a3c991105f 100644
--- a/ydb/core/ymq/actor/proxy_service.h
+++ b/ydb/core/ymq/actor/proxy_service.h
@@ -85,8 +85,8 @@ private:
void HandleGetLeaderNodeForQueueResponse(TSqsEvents::TEvGetLeaderNodeForQueueResponse::TPtr& ev);
private:
- TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCounters_;
- TIntrusivePtr<NMonitoring::TDynamicCounters> YmqPublicCounters_;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCounters_;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> YmqPublicCounters_;
/// A map of node ids to TNodeIfno
THashMap<ui32, TNodeInfoRef> NodesInfo_;
diff --git a/ydb/core/ymq/actor/purge.cpp b/ydb/core/ymq/actor/purge.cpp
index bf41126180..aff713d784 100644
--- a/ydb/core/ymq/actor/purge.cpp
+++ b/ydb/core/ymq/actor/purge.cpp
@@ -146,7 +146,7 @@ void TPurgeActor::MakeStage2Request(ui64 cleanupVersion, const TValue& messages,
if (status == TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecComplete) {
const TValue val(TValue::Create(ev.GetExecutionEngineEvaluatedResponse()));
const ui64 messagesDeleted = val["messagesDeleted"];
- ADD_COUNTER_COUPLE(Counters_, MessagesPurged, purged_count_per_second, messagesDeleted);
+ ADD_COUNTER_COUPLE(Counters_, MessagesPurged, purged_count_per_second, messagesDeleted);
RLOG_SQS_DEBUG("Purged " << messagesDeleted << " messages from queue [" << QueuePath_ << "]");
const bool versionIsSame = val["versionIsSame"];
if (versionIsSame) {
diff --git a/ydb/core/ymq/actor/queue_leader.cpp b/ydb/core/ymq/actor/queue_leader.cpp
index 335b4fa7f6..2c6e81d827 100644
--- a/ydb/core/ymq/actor/queue_leader.cpp
+++ b/ydb/core/ymq/actor/queue_leader.cpp
@@ -168,9 +168,9 @@ void TQueueLeader::PassAway() {
}
// Explicitly set absolute counters to zero for proper counting aggregated parent counters:
- SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, 0);
- SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, 0);
- SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0);
+ SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, 0);
+ SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, 0);
+ SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0);
TActorBootstrapped::PassAway();
}
@@ -1214,7 +1214,7 @@ void TQueueLeader::AnswerGetConfiguration(TSqsEvents::TEvGetConfiguration::TPtr&
auto resp = MakeHolder<TSqsEvents::TEvConfiguration>();
resp->RootUrl = RootUrl_;
- resp->SqsCoreCounters = Counters_->RootCounters.SqsCounters;
+ resp->SqsCoreCounters = Counters_->RootCounters.SqsCounters;
resp->QueueCounters = Counters_;
resp->UserCounters = UserCounters_;
resp->Shards = ShardsCount_;
@@ -1236,7 +1236,7 @@ void TQueueLeader::AnswerGetConfiguration(TSqsEvents::TEvGetConfiguration::TPtr&
void TQueueLeader::AnswerFailed(TSqsEvents::TEvGetConfiguration::TPtr& ev) {
auto answer = MakeHolder<TSqsEvents::TEvConfiguration>();
answer->RootUrl = RootUrl_;
- answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters;
+ answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters;
answer->QueueCounters = Counters_;
answer->UserCounters = UserCounters_;
answer->Fail = true;
@@ -1323,7 +1323,7 @@ void TQueueLeader::OnQueueConfiguration(const TSqsEvents::TEvExecuted::TRecord&
answer->UserExists = true;
answer->QueueExists = false;
answer->RootUrl = RootUrl_;
- answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters;
+ answer->SqsCoreCounters = Counters_->RootCounters.SqsCounters;
answer->QueueCounters = Counters_;
answer->UserCounters = UserCounters_;
answer->Fail = false;
@@ -1602,9 +1602,9 @@ void TQueueLeader::ReportMessagesCountMetricsIfReady() {
}
if (Counters_) {
- SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, messagesCount);
+ SET_COUNTER_COUPLE(Counters_, MessagesCount, stored_count, messagesCount);
if (!IsFifoQueue_) { // for fifo queues infly is always 0
- SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, inflyMessagesCount);
+ SET_COUNTER_COUPLE(Counters_, InflyMessagesCount, inflight_count, inflyMessagesCount);
}
}
}
@@ -1619,10 +1619,10 @@ void TQueueLeader::ReportOldestTimestampMetricsIfReady() {
}
if (Counters_) {
if (oldestMessagesTimestamp != Max<ui64>()) {
- auto age = (TActivationContext::Now() - TInstant::MilliSeconds(oldestMessagesTimestamp)).Seconds();
- SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, age);
+ auto age = (TActivationContext::Now() - TInstant::MilliSeconds(oldestMessagesTimestamp)).Seconds();
+ SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, age);
} else {
- SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0);
+ SET_COUNTER_COUPLE(Counters_, OldestMessageAgeSeconds, oldest_age_milliseconds, 0);
}
}
}
diff --git a/ydb/core/ymq/actor/queue_schema.cpp b/ydb/core/ymq/actor/queue_schema.cpp
index fe58018d74..91d3c5d3df 100644
--- a/ydb/core/ymq/actor/queue_schema.cpp
+++ b/ydb/core/ymq/actor/queue_schema.cpp
@@ -769,12 +769,12 @@ static const char* const CommitQueueParamsQuery = R"__(
(let queuesRow '(
'('Account userName)
'('QueueName name)))
-
- (let eventsRow '(
- '('Account userName)
- '('QueueName name)
- '('EventType (Uint64 '1))))
-
+
+ (let eventsRow '(
+ '('Account userName)
+ '('QueueName name)
+ '('EventType (Uint64 '1))))
+
(let queuesSelect '(
'QueueState
'QueueId
@@ -836,11 +836,11 @@ static const char* const CommitQueueParamsQuery = R"__(
'('MasterTabletId masterTabletId)
'('TablesFormat tablesFormat)))
- (let eventsUpdate '(
- '('CustomQueueName customName)
- '('EventTimestamp now)
- '('FolderId folderId)))
-
+ (let eventsUpdate '(
+ '('CustomQueueName customName)
+ '('EventTimestamp now)
+ '('FolderId folderId)))
+
(let attrRow '(%3$s))
(let attrUpdate '(
@@ -884,7 +884,7 @@ static const char* const CommitQueueParamsQuery = R"__(
(ListIf queueExists (SetResult 'meta queuesRead))
(ListIf willCommit (UpdateRow queuesTable queuesRow queuesUpdate))
- (ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate))
+ (ListIf willCommit (UpdateRow eventsTable eventsRow eventsUpdate))
(ListIf willCommit (UpdateRow attrsTable attrRow attrUpdate))
(If (Not willCommit) (AsList (Void))
@@ -1299,27 +1299,27 @@ static const char* EraseQueueRecordQuery = R"__(
(
(let name (Parameter 'NAME (DataType 'Utf8String)))
(let userName (Parameter 'USER_NAME (DataType 'Utf8String)))
- (let now (Parameter 'NOW (DataType 'Uint64)))
+ (let now (Parameter 'NOW (DataType 'Uint64)))
(let queuesTable '%2$s/.Queues)
- (let eventsTable '%2$s/.Events)
+ (let eventsTable '%2$s/.Events)
(let queuesRow '(
'('Account userName)
'('QueueName name)))
- (let eventsRow '(
- '('Account userName)
- '('QueueName name)
- '('EventType (Uint64 '0))))
-
+ (let eventsRow '(
+ '('Account userName)
+ '('QueueName name)
+ '('EventType (Uint64 '0))))
+
(let queuesSelect '(
'QueueState
'Version
'FifoQueue
- 'Shards
- 'CustomQueueName
- 'CreatedTimestamp
- 'FolderId))
+ 'Shards
+ 'CustomQueueName
+ 'CreatedTimestamp
+ 'FolderId))
(let queuesRead (SelectRow queuesTable queuesRow queuesSelect))
(let currentVersion
@@ -1329,28 +1329,28 @@ static const char* EraseQueueRecordQuery = R"__(
)
)
- (let queueCreateTs
- (Coalesce
- (Member queuesRead 'CreatedTimestamp)
- (Uint64 '0)
- )
- )
- (let folderId
- (Coalesce
- (Member queuesRead 'FolderId)
- (Utf8String '"")
- )
- )
-
- (let customName
- (Coalesce
- (Member queuesRead 'CustomQueueName)
- (Utf8String '"")
- )
- )
-
- (let eventTs (Max now (Add queueCreateTs (Uint64 '2))))
-
+ (let queueCreateTs
+ (Coalesce
+ (Member queuesRead 'CreatedTimestamp)
+ (Uint64 '0)
+ )
+ )
+ (let folderId
+ (Coalesce
+ (Member queuesRead 'FolderId)
+ (Utf8String '"")
+ )
+ )
+
+ (let customName
+ (Coalesce
+ (Member queuesRead 'CustomQueueName)
+ (Utf8String '"")
+ )
+ )
+
+ (let eventTs (Max now (Add queueCreateTs (Uint64 '2))))
+
(let queueExists
(Coalesce
(Or
@@ -1359,16 +1359,16 @@ static const char* EraseQueueRecordQuery = R"__(
)
(Bool 'false)))
- (let eventsUpdate '(
- '('CustomQueueName customName)
- '('EventTimestamp eventTs)
- '('FolderId folderId)))
-
+ (let eventsUpdate '(
+ '('CustomQueueName customName)
+ '('EventTimestamp eventTs)
+ '('FolderId folderId)))
+
(return (AsList
(SetResult 'exists queueExists)
(SetResult 'version currentVersion)
(SetResult 'fields queuesRead)
- (If queueExists (UpdateRow eventsTable eventsRow eventsUpdate) (Void))
+ (If queueExists (UpdateRow eventsTable eventsRow eventsUpdate) (Void))
(If queueExists (EraseRow queuesTable queuesRow) (Void))))
)
)__";
@@ -1376,13 +1376,13 @@ static const char* EraseQueueRecordQuery = R"__(
void TDeleteQueueSchemaActorV2::NextAction() {
switch (EDeleting(SI_)) {
case EDeleting::EraseQueueRecord: {
- auto ev = MakeExecuteEvent(Sprintf(EraseQueueRecordQuery, QueuePath_.GetUserPath().c_str(), Cfg().GetRoot().c_str()));
+ auto ev = MakeExecuteEvent(Sprintf(EraseQueueRecordQuery, QueuePath_.GetUserPath().c_str(), Cfg().GetRoot().c_str()));
auto* trans = ev->Record.MutableTransaction()->MutableMiniKQLTransaction();
- auto nowMs = TInstant::Now().MilliSeconds();
+ auto nowMs = TInstant::Now().MilliSeconds();
TParameters(trans->MutableParams()->MutableProto())
.Utf8("NAME", QueuePath_.QueueName)
- .Utf8("USER_NAME", QueuePath_.UserName)
- .Uint64("NOW", nowMs);
+ .Utf8("USER_NAME", QueuePath_.UserName)
+ .Uint64("NOW", nowMs);
Register(new TMiniKqlExecutionActor(SelfId(), RequestId_, std::move(ev), false, QueuePath_, GetTransactionCounters(UserCounters_)));
break;
diff --git a/ydb/core/ymq/actor/receive_message.cpp b/ydb/core/ymq/actor/receive_message.cpp
index 31fe47600e..2b5bf0e517 100644
--- a/ydb/core/ymq/actor/receive_message.cpp
+++ b/ydb/core/ymq/actor/receive_message.cpp
@@ -209,14 +209,14 @@ private:
// counters
const TDuration messageResideDuration = TActivationContext::Now() - message.SentTimestamp;
COLLECT_HISTOGRAM_COUNTER(QueueCounters_, MessageReceiveAttempts, message.ReceiveCount);
- COLLECT_HISTOGRAM_COUNTER(QueueCounters_, receive_attempts_count_rate, message.ReceiveCount);
+ COLLECT_HISTOGRAM_COUNTER(QueueCounters_, receive_attempts_count_rate, message.ReceiveCount);
COLLECT_HISTOGRAM_COUNTER(QueueCounters_, MessageReside_Duration, messageResideDuration.MilliSeconds());
- COLLECT_HISTOGRAM_COUNTER(QueueCounters_, reside_duration_milliseconds, messageResideDuration.MilliSeconds());
- INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_Count, received_count_per_second);
- ADD_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_BytesRead, received_bytes_per_second, message.Data.size());
+ COLLECT_HISTOGRAM_COUNTER(QueueCounters_, reside_duration_milliseconds, messageResideDuration.MilliSeconds());
+ INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_Count, received_count_per_second);
+ ADD_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_BytesRead, received_bytes_per_second, message.Data.size());
}
if (ev->Get()->Messages.empty()) {
- INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second);
+ INC_COUNTER_COUPLE(QueueCounters_, ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second);
}
}
SendReplyAndDie();
diff --git a/ydb/core/ymq/actor/send_message.cpp b/ydb/core/ymq/actor/send_message.cpp
index f2c84b5e59..e596b8f2e3 100644
--- a/ydb/core/ymq/actor/send_message.cpp
+++ b/ydb/core/ymq/actor/send_message.cpp
@@ -269,10 +269,10 @@ private:
// counters
if (status.Status == TSqsEvents::TEvSendMessageBatchResponse::ESendMessageStatus::AlreadySent) {
- INC_COUNTER_COUPLE(QueueCounters_, SendMessage_DeduplicationCount, deduplicated_count_per_second);
+ INC_COUNTER_COUPLE(QueueCounters_, SendMessage_DeduplicationCount, deduplicated_count_per_second);
} else {
- INC_COUNTER_COUPLE(QueueCounters_, SendMessage_Count, sent_count_per_second);
- ADD_COUNTER_COUPLE(QueueCounters_, SendMessage_BytesWritten, sent_bytes_per_second, CalculateMessageSize(*currentRequest));
+ INC_COUNTER_COUPLE(QueueCounters_, SendMessage_Count, sent_count_per_second);
+ ADD_COUNTER_COUPLE(QueueCounters_, SendMessage_BytesWritten, sent_bytes_per_second, CalculateMessageSize(*currentRequest));
}
} else {
MakeError(currentResponse, NErrors::INTERNAL_FAILURE);
diff --git a/ydb/core/ymq/actor/service.cpp b/ydb/core/ymq/actor/service.cpp
index 0ebf998f33..34c9a4ae46 100644
--- a/ydb/core/ymq/actor/service.cpp
+++ b/ydb/core/ymq/actor/service.cpp
@@ -11,7 +11,7 @@
#include "queues_list_reader.h"
#include "user_settings_names.h"
#include "user_settings_reader.h"
-#include "index_events_processor.h"
+#include "index_events_processor.h"
#include <ydb/public/lib/value/value.h>
#include <ydb/public/sdk/cpp/client/ydb_types/credentials/credentials.h>
@@ -28,11 +28,11 @@
#include <library/cpp/actors/core/events.h>
#include <library/cpp/actors/core/hfunc.h>
-#include <library/cpp/logger/global/global.h>
+#include <library/cpp/logger/global/global.h>
#include <util/generic/algorithm.h>
#include <util/generic/hash_set.h>
-#include <util/stream/file.h>
+#include <util/stream/file.h>
#include <util/string/builder.h>
#include <util/string/cast.h>
#include <util/system/hostname.h>
@@ -63,17 +63,17 @@ constexpr ui64 LIST_QUEUES_WAKEUP_TAG = 2;
constexpr size_t EARLY_REQUEST_USERS_LIST_MAX_BUDGET = 10;
constexpr i64 EARLY_REQUEST_QUEUES_LIST_MAX_BUDGET = 5; // per user
-bool IsInternalFolder(const TString& folder) {
- return folder.StartsWith(".sys");
-}
-
+bool IsInternalFolder(const TString& folder) {
+ return folder.StartsWith(".sys");
+}
+
struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> {
- TQueueInfo(
+ TQueueInfo(
TString userName, TString queueName, TString rootUrl, ui64 leaderTabletId, TString customName,
- TString folderId, ui64 version, ui64 shardsCount, const TIntrusivePtr<TUserCounters>& userCounters,
- const TActorId& schemeCache, TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResourcesForUser,
- bool insertCounters
- )
+ TString folderId, ui64 version, ui64 shardsCount, const TIntrusivePtr<TUserCounters>& userCounters,
+ const TActorId& schemeCache, TIntrusivePtr<TSqsEvents::TQuoterResourcesForActions> quoterResourcesForUser,
+ bool insertCounters
+ )
: UserName_(std::move(userName))
, QueueName_(std::move(queueName))
, CustomName_(std::move(customName))
@@ -128,16 +128,16 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> {
LWPROBE(CreateLeader, UserName_, QueueName_, reason);
LocalLeader_ = TActivationContext::Register(new TQueueLeader(UserName_, QueueName_, FolderId_, RootUrl_, Counters_, UserCounters_, SchemeCache_, QuoterResourcesForUser_));
LOG_SQS_INFO("Start local leader [" << UserName_ << "/" << QueueName_ << "] actor " << LocalLeader_);
-
- // ToDo: Should better make TFolderCounters struct and move it there.
- // Will have to refactor TQueueCounters a bit, since it directly works with TUserCounters
- if (FolderId_) {
- auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_);
- if (folderCounters.YmqCounters) {
- auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false);
- (*counter)++;
- }
- }
+
+ // ToDo: Should better make TFolderCounters struct and move it there.
+ // Will have to refactor TQueueCounters a bit, since it directly works with TUserCounters
+ if (FolderId_) {
+ auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_);
+ if (folderCounters.YmqCounters) {
+ auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false);
+ (*counter)++;
+ }
+ }
}
}
@@ -148,16 +148,16 @@ struct TSqsService::TQueueInfo : public TAtomicRefCount<TQueueInfo> {
LOG_SQS_INFO("Stop local leader [" << UserName_ << "/" << QueueName_ << "] actor " << LocalLeader_);
TActivationContext::Send(new IEventHandle(LocalLeader_, SelfId(), new TEvPoisonPill()));
LocalLeader_ = TActorId();
- if (FolderId_) {
- auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_);
- if (folderCounters.YmqCounters) {
- auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false);
- counter->Dec();
- if (counter->Val() == 0) {
- RemoveFolderCounters(UserCounters_->UserCounters, FolderId_);
- }
- }
- }
+ if (FolderId_) {
+ auto folderCounters = GetFolderCounters(UserCounters_->UserCounters, FolderId_);
+ if (folderCounters.YmqCounters) {
+ auto counter = folderCounters.YmqCounters->GetCounter("queue.total_count", false);
+ counter->Dec();
+ if (counter->Val() == 0) {
+ RemoveFolderCounters(UserCounters_->UserCounters, FolderId_);
+ }
+ }
+ }
}
}
@@ -275,10 +275,10 @@ static TString GetEndpoint(const NKikimrConfig::TSqsConfig& config) {
}
}
-TSqsService::TSqsService(const TMaybe<ui32>& ydbPort) {
- if (ydbPort.Defined()) {
- YcSearchEventsConfig.GrpcPort = *ydbPort;
- }
+TSqsService::TSqsService(const TMaybe<ui32>& ydbPort) {
+ if (ydbPort.Defined()) {
+ YcSearchEventsConfig.GrpcPort = *ydbPort;
+ }
DebugInfo->SqsServiceActorPtr = this;
}
TSqsService::~TSqsService() {
@@ -298,11 +298,11 @@ void TSqsService::Bootstrap() {
// Counters.
SqsCoreCounters_ = GetSqsServiceCounters(AppData()->Counters, "core");
- YmqRootCounters_ = GetYmqPublicCounters(AppData()->Counters);
+ YmqRootCounters_ = GetYmqPublicCounters(AppData()->Counters);
AllocPoolCounters_ = std::make_shared<TAlignedPagePoolCounters>(AppData()->Counters, "sqs");
- AggregatedUserCounters_ = MakeIntrusive<TUserCounters>(
- Cfg(), SqsCoreCounters_, nullptr, AllocPoolCounters_, TOTAL_COUNTER_LABEL, nullptr, true
- );
+ AggregatedUserCounters_ = MakeIntrusive<TUserCounters>(
+ Cfg(), SqsCoreCounters_, nullptr, AllocPoolCounters_, TOTAL_COUNTER_LABEL, nullptr, true
+ );
AggregatedUserCounters_->ShowDetailedCounters(TInstant::Max());
InitSchemeCache();
@@ -314,29 +314,29 @@ void TSqsService::Bootstrap() {
RequestSqsUsersList();
RequestSqsQueuesList();
-
- if (Cfg().HasYcSearchEventsConfig() && YcSearchEventsConfig.GrpcPort) {
- auto& ycSearchCfg = Cfg().GetYcSearchEventsConfig();
- YcSearchEventsConfig.Enabled = ycSearchCfg.GetEnableYcSearch();
-
- YcSearchEventsConfig.ReindexInterval = TDuration::Seconds(ycSearchCfg.GetReindexIntervalSeconds());
- YcSearchEventsConfig.RescanInterval = TDuration::Seconds(ycSearchCfg.GetRescanIntervalSeconds());
-
- auto driverConfig = NYdb::TDriverConfig().SetEndpoint(
- TStringBuilder() << "localhost:" << YcSearchEventsConfig.GrpcPort);
- if (ycSearchCfg.HasTenantMode() && ycSearchCfg.GetTenantMode()) {
- driverConfig.SetDatabase(Cfg().GetRoot());
- YcSearchEventsConfig.TenantMode = true;
- }
+
+ if (Cfg().HasYcSearchEventsConfig() && YcSearchEventsConfig.GrpcPort) {
+ auto& ycSearchCfg = Cfg().GetYcSearchEventsConfig();
+ YcSearchEventsConfig.Enabled = ycSearchCfg.GetEnableYcSearch();
+
+ YcSearchEventsConfig.ReindexInterval = TDuration::Seconds(ycSearchCfg.GetReindexIntervalSeconds());
+ YcSearchEventsConfig.RescanInterval = TDuration::Seconds(ycSearchCfg.GetRescanIntervalSeconds());
+
+ auto driverConfig = NYdb::TDriverConfig().SetEndpoint(
+ TStringBuilder() << "localhost:" << YcSearchEventsConfig.GrpcPort);
+ if (ycSearchCfg.HasTenantMode() && ycSearchCfg.GetTenantMode()) {
+ driverConfig.SetDatabase(Cfg().GetRoot());
+ YcSearchEventsConfig.TenantMode = true;
+ }
auto factory = AppData()->SqsAuthFactory;
Y_VERIFY(factory);
-
+
driverConfig.SetCredentialsProviderFactory(factory->CreateCredentialsProviderFactory(Cfg()));
-
- YcSearchEventsConfig.Driver = MakeHolder<NYdb::TDriver>(driverConfig);
- MakeAndRegisterYcEventsProcessor();
- }
+
+ YcSearchEventsConfig.Driver = MakeHolder<NYdb::TDriver>(driverConfig);
+ MakeAndRegisterYcEventsProcessor();
+ }
}
STATEFN(TSqsService::StateFunc) {
@@ -947,18 +947,18 @@ TSqsService::TUsersMap::iterator TSqsService::MutableUserIter(const TString& use
auto userIt = Users_.find(userName);
if (userIt == Users_.end()) {
LOG_SQS_INFO("Creating user info record for user [" << userName << "]");
- bool isInternal = IsInternalFolder(userName);
- if (isInternal) {
- LOG_SQS_INFO("[" << userName << "] is considered and internal service folder, will not create YMQ counters");
- }
- TUserInfoPtr user = new TUserInfo(
- userName,
- new TUserCounters(
- Cfg(), SqsCoreCounters_,
- isInternal ? nullptr : YmqRootCounters_,
- AllocPoolCounters_, userName, AggregatedUserCounters_, false
- )
- );
+ bool isInternal = IsInternalFolder(userName);
+ if (isInternal) {
+ LOG_SQS_INFO("[" << userName << "] is considered and internal service folder, will not create YMQ counters");
+ }
+ TUserInfoPtr user = new TUserInfo(
+ userName,
+ new TUserCounters(
+ Cfg(), SqsCoreCounters_,
+ isInternal ? nullptr : YmqRootCounters_,
+ AllocPoolCounters_, userName, AggregatedUserCounters_, false
+ )
+ );
user->InitQuoterResources();
userIt = Users_.emplace(userName, user).first;
@@ -1024,7 +1024,7 @@ void TSqsService::RemoveQueue(const TString& userName, const TString& queue) {
queuePtr->GetLeaderNodeRequests_.clear();
LeaderTabletIdToQueue_.erase(queuePtr->LeaderTabletId_);
userIt->second->QueueByNameAndFolder_.erase(std::make_pair(queuePtr->CustomName_, queuePtr->FolderId_));
-
+
userIt->second->Queues_.erase(queueIt);
queuePtr->Counters_->RemoveCounters();
}
@@ -1290,13 +1290,13 @@ void TSqsService::AnswerErrorToRequests(const TUserInfoPtr& user, TMultimap& map
}
map.clear();
}
-
-void TSqsService::MakeAndRegisterYcEventsProcessor() {
- if (!YcSearchEventsConfig.Enabled)
- return;
-
- auto root = YcSearchEventsConfig.TenantMode ? TString() : Cfg().GetRoot();
-
+
+void TSqsService::MakeAndRegisterYcEventsProcessor() {
+ if (!YcSearchEventsConfig.Enabled)
+ return;
+
+ auto root = YcSearchEventsConfig.TenantMode ? TString() : Cfg().GetRoot();
+
auto factory = AppData()->SqsEventsWriterFactory;
Y_VERIFY(factory);
Register(new TSearchEventsProcessor(
@@ -1304,14 +1304,14 @@ void TSqsService::MakeAndRegisterYcEventsProcessor() {
MakeSimpleShared<NYdb::NTable::TTableClient>(*YcSearchEventsConfig.Driver),
factory->CreateEventsWriter(Cfg(), GetSqsServiceCounters(AppData()->Counters, "yc_unified_agent"))
));
-}
-//
-//IActor* CreateSqsService(const TYcSearchEventsConfig& ycSearchEventsConfig) {
-// return new TSqsService(ycSearchEventsConfig);
-//}
-
-IActor* CreateSqsService(TMaybe<ui32> ydbPort) {
- return new TSqsService(ydbPort);
+}
+//
+//IActor* CreateSqsService(const TYcSearchEventsConfig& ycSearchEventsConfig) {
+// return new TSqsService(ycSearchEventsConfig);
+//}
+
+IActor* CreateSqsService(TMaybe<ui32> ydbPort) {
+ return new TSqsService(ydbPort);
}
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/service.h b/ydb/core/ymq/actor/service.h
index 5a0ad1294f..caf091c24b 100644
--- a/ydb/core/ymq/actor/service.h
+++ b/ydb/core/ymq/actor/service.h
@@ -3,7 +3,7 @@
#include "events.h"
#include "log.h"
#include "serviceid.h"
-#include "index_events_processor.h"
+#include "index_events_processor.h"
#include <ydb/core/ymq/base/events_writer_iface.h>
#include <ydb/core/base/tablet_pipe.h>
@@ -14,7 +14,7 @@
#include <library/cpp/actors/core/actor_bootstrapped.h>
#include <util/generic/hash.h>
#include <util/generic/ptr.h>
-#include <library/cpp/logger/log.h>
+#include <library/cpp/logger/log.h>
namespace NKikimr::NSQS {
@@ -22,7 +22,7 @@ class TSqsService
: public TActorBootstrapped<TSqsService>
{
public:
- TSqsService(const TMaybe<ui32>& ydbPort);
+ TSqsService(const TMaybe<ui32>& ydbPort);
~TSqsService();
void Bootstrap();
@@ -129,12 +129,12 @@ private:
void NotifyLocalDeadLetterQueuesLeaders(const std::vector<TSqsEvents::TEvQueuesList::TQueueRecord>& sortedQueues) const;
- void MakeAndRegisterYcEventsProcessor();
-
+ void MakeAndRegisterYcEventsProcessor();
+
private:
TString RootUrl_;
TIntrusivePtr<NMonitoring::TDynamicCounters> SqsCoreCounters_;
- TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_;
+ TIntrusivePtr<NMonitoring::TDynamicCounters> YmqRootCounters_;
std::shared_ptr<TAlignedPagePoolCounters> AllocPoolCounters_;
TIntrusivePtr<TUserCounters> AggregatedUserCounters_;
TUsersMap Users_;
@@ -158,17 +158,17 @@ private:
THashMultiMap<TString, TSqsEvents::TEvGetQueueId::TPtr> GetQueueIdRequests_; // user name -> request
THashMultiMap<TString, TSqsEvents::TEvGetQueueFolderIdAndCustomName::TPtr> GetQueueFolderIdAndCustomNameRequests_; // user name -> request
THashMultiMap<TString, TSqsEvents::TEvCountQueues::TPtr> CountQueuesRequests_; // user name -> request
-
-
- struct TYcSearchEventsConfig {
- THolder<NYdb::TDriver> Driver;
- ui32 GrpcPort = 0;
- bool Enabled = false;
- bool TenantMode = false;
- TDuration ReindexInterval = TDuration::Hours(4);
- TDuration RescanInterval = TDuration::Minutes(1);
- };
- TYcSearchEventsConfig YcSearchEventsConfig;
+
+
+ struct TYcSearchEventsConfig {
+ THolder<NYdb::TDriver> Driver;
+ ui32 GrpcPort = 0;
+ bool Enabled = false;
+ bool TenantMode = false;
+ TDuration ReindexInterval = TDuration::Hours(4);
+ TDuration RescanInterval = TDuration::Minutes(1);
+ };
+ TYcSearchEventsConfig YcSearchEventsConfig;
};
} // namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/serviceid.h b/ydb/core/ymq/actor/serviceid.h
index 741b094c9c..7219d918ba 100644
--- a/ydb/core/ymq/actor/serviceid.h
+++ b/ydb/core/ymq/actor/serviceid.h
@@ -31,7 +31,7 @@ inline TActorId MakeSqsMeteringServiceID() {
return TActorId(0, TStringBuf("SQS_METER"));
}
-IActor* CreateSqsService(TMaybe<ui32> ydbPort = Nothing());
+IActor* CreateSqsService(TMaybe<ui32> ydbPort = Nothing());
IActor* CreateSqsProxyService();
IActor* CreateSqsAccessService(const TString& address, const TString& pathToRootCA);
IActor* CreateSqsFolderService(const TString& address, const TString& pathToRootCA);
diff --git a/ydb/core/ymq/actor/ya.make b/ydb/core/ymq/actor/ya.make
index 5bbd51aa09..070653bb57 100644
--- a/ydb/core/ymq/actor/ya.make
+++ b/ydb/core/ymq/actor/ya.make
@@ -23,7 +23,7 @@ SRCS(
garbage_collector.cpp
get_queue_attributes.cpp
get_queue_url.cpp
- index_events_processor.cpp
+ index_events_processor.cpp
infly.cpp
log.cpp
list_dead_letter_source_queues.cpp
diff --git a/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp b/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp
index 77863e5a7e..e484f117be 100644
--- a/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp
+++ b/ydb/core/ymq/actor/yc_search_ut/index_events_processor_ut.cpp
@@ -1,379 +1,379 @@
#include <ydb/core/testlib/test_client.h>
#include <ydb/core/ymq/actor/index_events_processor.h>
-#include <library/cpp/testing/unittest/registar.h>
-#include <util/stream/file.h>
-#include <library/cpp/json/json_reader.h>
-#include "test_events_writer.h"
-
-namespace NKikimr::NSQS {
-
-using namespace Tests;
-using namespace NYdb;
-
-
-class TIndexProcesorTests : public TTestBase {
- using EEvType = TSearchEventsProcessor::EQueueEventType;
-
-public:
- TIndexProcesorTests() {
- TPortManager portManager;
- auto mbusPort = portManager.GetPort(2134);
- auto grpcPort = portManager.GetPort(2135);
- auto settings = TServerSettings(mbusPort);
- settings.SetDomainName("Root");
- Server = MakeHolder<TServer>(settings);
+#include <library/cpp/testing/unittest/registar.h>
+#include <util/stream/file.h>
+#include <library/cpp/json/json_reader.h>
+#include "test_events_writer.h"
+
+namespace NKikimr::NSQS {
+
+using namespace Tests;
+using namespace NYdb;
+
+
+class TIndexProcesorTests : public TTestBase {
+ using EEvType = TSearchEventsProcessor::EQueueEventType;
+
+public:
+ TIndexProcesorTests() {
+ TPortManager portManager;
+ auto mbusPort = portManager.GetPort(2134);
+ auto grpcPort = portManager.GetPort(2135);
+ auto settings = TServerSettings(mbusPort);
+ settings.SetDomainName("Root");
+ Server = MakeHolder<TServer>(settings);
Server->EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(grpcPort));
- auto driverConfig = TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort);
-
- Driver = MakeHolder<TDriver>(driverConfig);
+ auto driverConfig = TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << grpcPort);
+
+ Driver = MakeHolder<TDriver>(driverConfig);
TableClient = MakeSimpleShared<NYdb::NTable::TTableClient>(*Driver);
- }
-
-private:
- struct TTestRunner {
- TString TestName;
- TString SchemePath;
- TSearchEventsProcessor* Processor;
- TDispatchOptions DispatchOpts;
- TInstant CurrTs;
- TInstant PrevTs;
- TMockSessionPtr EventsWriter;
- TIndexProcesorTests* Parent;
- TActorId ProcessorId;
- TTestRunner(const TString& name, TIndexProcesorTests* parent)
- : TestName(name)
- , SchemePath(parent->Root + "/" + name)
- , Processor()
- , CurrTs(TInstant::Now())
- , PrevTs(TInstant::Seconds(CurrTs.Seconds() - 1))
- , Parent(parent)
- {
- EventsWriter = MakeIntrusive<TTestEventsWriter>();
-
- auto* runtime = parent->Server->GetRuntime();
- Processor = new TSearchEventsProcessor(
- SchemePath,
- TDuration::Minutes(10), TDuration::MilliSeconds(100),
- parent->TableClient,
- EventsWriter,
- true
- );
- ProcessorId = runtime->Register(Processor);
- runtime->EnableScheduleForActor(ProcessorId, true);
- parent->Server->GetRuntime()->SetObserverFunc(
- [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>&)
- {
- if (Processor->GetReindexCount() >= 1) return TTestActorRuntimeBase::EEventAction::DROP;
- else return TTestActorRuntimeBase::EEventAction::PROCESS;
- }
- );
- InitTables();
- }
- ~TTestRunner() {
- if (Processor != nullptr) {
- auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StopAllTag));
- Parent->Server->GetRuntime()->Send(handle);
- }
- }
- void InitTables() {
- TClient client(Parent->Server->GetSettings());
- client.MkDir(Parent->Root, SchemePath);
- auto session = Parent->TableClient->CreateSession().GetValueSync().GetSession();
+ }
+
+private:
+ struct TTestRunner {
+ TString TestName;
+ TString SchemePath;
+ TSearchEventsProcessor* Processor;
+ TDispatchOptions DispatchOpts;
+ TInstant CurrTs;
+ TInstant PrevTs;
+ TMockSessionPtr EventsWriter;
+ TIndexProcesorTests* Parent;
+ TActorId ProcessorId;
+ TTestRunner(const TString& name, TIndexProcesorTests* parent)
+ : TestName(name)
+ , SchemePath(parent->Root + "/" + name)
+ , Processor()
+ , CurrTs(TInstant::Now())
+ , PrevTs(TInstant::Seconds(CurrTs.Seconds() - 1))
+ , Parent(parent)
+ {
+ EventsWriter = MakeIntrusive<TTestEventsWriter>();
+
+ auto* runtime = parent->Server->GetRuntime();
+ Processor = new TSearchEventsProcessor(
+ SchemePath,
+ TDuration::Minutes(10), TDuration::MilliSeconds(100),
+ parent->TableClient,
+ EventsWriter,
+ true
+ );
+ ProcessorId = runtime->Register(Processor);
+ runtime->EnableScheduleForActor(ProcessorId, true);
+ parent->Server->GetRuntime()->SetObserverFunc(
+ [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle>&)
+ {
+ if (Processor->GetReindexCount() >= 1) return TTestActorRuntimeBase::EEventAction::DROP;
+ else return TTestActorRuntimeBase::EEventAction::PROCESS;
+ }
+ );
+ InitTables();
+ }
+ ~TTestRunner() {
+ if (Processor != nullptr) {
+ auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StopAllTag));
+ Parent->Server->GetRuntime()->Send(handle);
+ }
+ }
+ void InitTables() {
+ TClient client(Parent->Server->GetSettings());
+ client.MkDir(Parent->Root, SchemePath);
+ auto session = Parent->TableClient->CreateSession().GetValueSync().GetSession();
auto desc = NYdb::NTable::TTableBuilder()
- .AddNullableColumn("Account", EPrimitiveType::Utf8)
- .AddNullableColumn("QueueName", EPrimitiveType::Utf8)
- .AddNullableColumn("CreatedTimestamp", EPrimitiveType::Uint64)
- .AddNullableColumn("FolderId", EPrimitiveType::Utf8)
- .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8)
- .SetPrimaryKeyColumns({"Account", "QueueName"})
- .Build();
+ .AddNullableColumn("Account", EPrimitiveType::Utf8)
+ .AddNullableColumn("QueueName", EPrimitiveType::Utf8)
+ .AddNullableColumn("CreatedTimestamp", EPrimitiveType::Uint64)
+ .AddNullableColumn("FolderId", EPrimitiveType::Utf8)
+ .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8)
+ .SetPrimaryKeyColumns({"Account", "QueueName"})
+ .Build();
auto eventsDesc = NYdb::NTable::TTableBuilder()
- .AddNullableColumn("Account", EPrimitiveType::Utf8)
- .AddNullableColumn("QueueName", EPrimitiveType::Utf8)
- .AddNullableColumn("EventType", EPrimitiveType::Uint64)
- .AddNullableColumn("EventTimestamp", EPrimitiveType::Uint64)
- .AddNullableColumn("FolderId", EPrimitiveType::Utf8)
- .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8)
- .SetPrimaryKeyColumns({"Account", "QueueName", "EventType"})
- .Build();
- auto f1 = session.CreateTable(SchemePath + "/.Queues", std::move(desc));
- auto f2 = session.CreateTable(SchemePath + "/.Events", std::move(eventsDesc));
-
- auto status = f1.GetValueSync();
- UNIT_ASSERT(status.IsSuccess());
- status = f2.GetValueSync();
- UNIT_ASSERT(status.IsSuccess());
- session.Close();
- }
-
- TAsyncStatus RunDataQuery(const TString& query) {
+ .AddNullableColumn("Account", EPrimitiveType::Utf8)
+ .AddNullableColumn("QueueName", EPrimitiveType::Utf8)
+ .AddNullableColumn("EventType", EPrimitiveType::Uint64)
+ .AddNullableColumn("EventTimestamp", EPrimitiveType::Uint64)
+ .AddNullableColumn("FolderId", EPrimitiveType::Utf8)
+ .AddNullableColumn("CustomQueueName", EPrimitiveType::Utf8)
+ .SetPrimaryKeyColumns({"Account", "QueueName", "EventType"})
+ .Build();
+ auto f1 = session.CreateTable(SchemePath + "/.Queues", std::move(desc));
+ auto f2 = session.CreateTable(SchemePath + "/.Events", std::move(eventsDesc));
+
+ auto status = f1.GetValueSync();
+ UNIT_ASSERT(status.IsSuccess());
+ status = f2.GetValueSync();
+ UNIT_ASSERT(status.IsSuccess());
+ session.Close();
+ }
+
+ TAsyncStatus RunDataQuery(const TString& query) {
auto status = Parent->TableClient->RetryOperation<NYdb::NTable::TDataQueryResult>(
[query](NYdb::NTable::TSession session) {
- return session.ExecuteDataQuery(
+ return session.ExecuteDataQuery(
query, NYdb::NTable::TTxControl::BeginTx().CommitTx(),
NYdb::NTable::TExecDataQuerySettings().ClientTimeout(TDuration::Seconds(10))
).Apply([](const auto &future) mutable {
- return future;
- });
- });
- return status;
- }
-
- void ExecDataQuery(const TString& query) {
- Cerr << "===Execute query: " << query << Endl;
- auto statusVal = RunDataQuery(query).GetValueSync();
- if (!statusVal.IsSuccess()) {
- Cerr << "Query execution failed with error: " << statusVal.GetIssues().ToString() << Endl;
- }
- UNIT_ASSERT(statusVal.IsSuccess());
- }
- void AddEvent(
- const TString& account, const TString& queueName, const EEvType& type, TInstant ts = TInstant::Zero())
- {
- if (ts == TInstant::Zero())
- ts = CurrTs;
- TStringBuilder queryBuilder;
- queryBuilder << "UPSERT INTO [" << SchemePath << "/.Events] (Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId) "
- << "VALUES (\"" << account << "\", \"" << queueName << "\", " << static_cast<ui64>(type) << ", \"myQueueCustomName\", "
- << ts.MilliSeconds() << ", \"myFolder\");";
- ExecDataQuery(queryBuilder.c_str());
- }
-
- void AddQueue(const TString& account, const TString& queueName, TInstant ts = TInstant::Zero()) {
- if (ts == TInstant::Zero())
- ts = CurrTs;
- TStringBuilder queryBuilder;
- queryBuilder << "UPSERT INTO [" << SchemePath << "/.Queues] (Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) "
- << "VALUES (\"" << account << "\", \"" << queueName << "\", \"myQueueCustomName\", "
- << ts.MilliSeconds() << ", \"myFolder\");";
- ExecDataQuery(queryBuilder.c_str());
- }
-
- void AddQueuesBatch(const TString& account, const TString& queueNameBase, ui64 count, ui64 startIndex = 0) {
- Cerr << "===Started add queue batch\n";
+ return future;
+ });
+ });
+ return status;
+ }
+
+ void ExecDataQuery(const TString& query) {
+ Cerr << "===Execute query: " << query << Endl;
+ auto statusVal = RunDataQuery(query).GetValueSync();
+ if (!statusVal.IsSuccess()) {
+ Cerr << "Query execution failed with error: " << statusVal.GetIssues().ToString() << Endl;
+ }
+ UNIT_ASSERT(statusVal.IsSuccess());
+ }
+ void AddEvent(
+ const TString& account, const TString& queueName, const EEvType& type, TInstant ts = TInstant::Zero())
+ {
+ if (ts == TInstant::Zero())
+ ts = CurrTs;
+ TStringBuilder queryBuilder;
+ queryBuilder << "UPSERT INTO [" << SchemePath << "/.Events] (Account, QueueName, EventType, CustomQueueName, EventTimestamp, FolderId) "
+ << "VALUES (\"" << account << "\", \"" << queueName << "\", " << static_cast<ui64>(type) << ", \"myQueueCustomName\", "
+ << ts.MilliSeconds() << ", \"myFolder\");";
+ ExecDataQuery(queryBuilder.c_str());
+ }
+
+ void AddQueue(const TString& account, const TString& queueName, TInstant ts = TInstant::Zero()) {
+ if (ts == TInstant::Zero())
+ ts = CurrTs;
+ TStringBuilder queryBuilder;
+ queryBuilder << "UPSERT INTO [" << SchemePath << "/.Queues] (Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) "
+ << "VALUES (\"" << account << "\", \"" << queueName << "\", \"myQueueCustomName\", "
+ << ts.MilliSeconds() << ", \"myFolder\");";
+ ExecDataQuery(queryBuilder.c_str());
+ }
+
+ void AddQueuesBatch(const TString& account, const TString& queueNameBase, ui64 count, ui64 startIndex = 0) {
+ Cerr << "===Started add queue batch\n";
TDeque<NYdb::NTable::TAsyncDataQueryResult> results;
- ui64 maxInflight = 1;
- auto getRessionResult = Parent->TableClient->GetSession().GetValueSync();
- UNIT_ASSERT(getRessionResult.IsSuccess());
- auto session = getRessionResult.GetSession();
- TStringBuilder queryBuilder;
- queryBuilder << "DECLARE $QueueName as Utf8; "
- << "UPSERT INTO [" << SchemePath << "/.Queues] "
- << "(Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) "
- << "VALUES (\"" << account << "\", $QueueName, \"myQueueCustomName\", "
- << CurrTs.MilliSeconds() << ", \"myFolder\");";
-
- auto preparedResult = session.PrepareDataQuery(queryBuilder.c_str()).GetValueSync();
- UNIT_ASSERT(preparedResult.IsSuccess());
- auto prepQuery = preparedResult.GetQuery();
-
- auto txResult = session.BeginTransaction().GetValueSync();
- UNIT_ASSERT(txResult.IsSuccess());
- auto tx = txResult.GetTransaction();
+ ui64 maxInflight = 1;
+ auto getRessionResult = Parent->TableClient->GetSession().GetValueSync();
+ UNIT_ASSERT(getRessionResult.IsSuccess());
+ auto session = getRessionResult.GetSession();
+ TStringBuilder queryBuilder;
+ queryBuilder << "DECLARE $QueueName as Utf8; "
+ << "UPSERT INTO [" << SchemePath << "/.Queues] "
+ << "(Account, QueueName, CustomQueueName, CreatedTimestamp, FolderId) "
+ << "VALUES (\"" << account << "\", $QueueName, \"myQueueCustomName\", "
+ << CurrTs.MilliSeconds() << ", \"myFolder\");";
+
+ auto preparedResult = session.PrepareDataQuery(queryBuilder.c_str()).GetValueSync();
+ UNIT_ASSERT(preparedResult.IsSuccess());
+ auto prepQuery = preparedResult.GetQuery();
+
+ auto txResult = session.BeginTransaction().GetValueSync();
+ UNIT_ASSERT(txResult.IsSuccess());
+ auto tx = txResult.GetTransaction();
auto txControl = NYdb::NTable::TTxControl::Tx(tx);
-
- for (auto i = startIndex; i < startIndex + count; i++) {
- while (results.size() < maxInflight) {
- auto builder = prepQuery.GetParamsBuilder();
- builder.AddParam("$QueueName").Utf8(queueNameBase + ToString(i)).Build();
-
- results.push_back(
- prepQuery.Execute(txControl, builder.Build())
- );
- }
- results.front().Wait();
- while(!results.empty() && results.front().HasValue()) {
- UNIT_ASSERT_C(results.front().GetValueSync().IsSuccess(),results.front().GetValueSync().GetIssues().ToString().c_str());
- results.pop_front();
- }
-
- }
- while (!results.empty()) {
- auto statusVal = results.front().GetValueSync();
- UNIT_ASSERT(statusVal.IsSuccess());
- results.pop_front();
- }
- auto commitStatus = tx.Commit().GetValueSync();
- UNIT_ASSERT(commitStatus.IsSuccess());
- }
-
- ui64 CountRecords(const TString& table) const {
- TStringBuilder queryBuilder;
- TMaybe<NYdb::NTable::TDataQueryResult> countEventsResult;
-
- queryBuilder << "select count(*) as Count from [" << SchemePath << "/." << table << "];";
+
+ for (auto i = startIndex; i < startIndex + count; i++) {
+ while (results.size() < maxInflight) {
+ auto builder = prepQuery.GetParamsBuilder();
+ builder.AddParam("$QueueName").Utf8(queueNameBase + ToString(i)).Build();
+
+ results.push_back(
+ prepQuery.Execute(txControl, builder.Build())
+ );
+ }
+ results.front().Wait();
+ while(!results.empty() && results.front().HasValue()) {
+ UNIT_ASSERT_C(results.front().GetValueSync().IsSuccess(),results.front().GetValueSync().GetIssues().ToString().c_str());
+ results.pop_front();
+ }
+
+ }
+ while (!results.empty()) {
+ auto statusVal = results.front().GetValueSync();
+ UNIT_ASSERT(statusVal.IsSuccess());
+ results.pop_front();
+ }
+ auto commitStatus = tx.Commit().GetValueSync();
+ UNIT_ASSERT(commitStatus.IsSuccess());
+ }
+
+ ui64 CountRecords(const TString& table) const {
+ TStringBuilder queryBuilder;
+ TMaybe<NYdb::NTable::TDataQueryResult> countEventsResult;
+
+ queryBuilder << "select count(*) as Count from [" << SchemePath << "/." << table << "];";
auto status = Parent->TableClient->RetryOperation<NYdb::NTable::TDataQueryResult>([query = TString(queryBuilder), &countEventsResult](NYdb::NTable::TSession session) {
- return session.ExecuteDataQuery(
+ return session.ExecuteDataQuery(
query, NYdb::NTable::TTxControl::BeginTx().CommitTx(),
NYdb::NTable::TExecDataQuerySettings().ClientTimeout(TDuration::Seconds(5))
- ).Apply([&](const auto& future) mutable {
- countEventsResult = future.GetValue();
- return future;
- });
- }).GetValueSync();
- UNIT_ASSERT(status.IsSuccess());
- auto result = countEventsResult.Get()->GetResultSet(0);
- TResultSetParser parser(result);
- auto haveNext = parser.TryNextRow();
- UNIT_ASSERT(haveNext);
- return parser.ColumnParser("Count").GetUint64();
- }
- ui64 CountEvents() const {
- return CountRecords("Events");
- }
- ui64 CountQueues() const {
- return CountRecords("Queues");
- }
- void DispatchEvents() {
- auto initialCount = Processor->GetReindexCount();
- DispatchOpts.CustomFinalCondition = [initialCount, processor = Processor]() { return processor->GetReindexCount() > initialCount; };
- if (!initialCount) {
- auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StartQueuesListingTag));
- Parent->Server->GetRuntime()->Send(handle);
- }
- Parent->Server->GetRuntime()->DispatchEvents(DispatchOpts, TDuration::Minutes(1));
- }
- };
- void SetUp() override {
- TClient client(Server->GetSettings());
+ ).Apply([&](const auto& future) mutable {
+ countEventsResult = future.GetValue();
+ return future;
+ });
+ }).GetValueSync();
+ UNIT_ASSERT(status.IsSuccess());
+ auto result = countEventsResult.Get()->GetResultSet(0);
+ TResultSetParser parser(result);
+ auto haveNext = parser.TryNextRow();
+ UNIT_ASSERT(haveNext);
+ return parser.ColumnParser("Count").GetUint64();
+ }
+ ui64 CountEvents() const {
+ return CountRecords("Events");
+ }
+ ui64 CountQueues() const {
+ return CountRecords("Queues");
+ }
+ void DispatchEvents() {
+ auto initialCount = Processor->GetReindexCount();
+ DispatchOpts.CustomFinalCondition = [initialCount, processor = Processor]() { return processor->GetReindexCount() > initialCount; };
+ if (!initialCount) {
+ auto handle = new IEventHandle(ProcessorId, TActorId(), new TEvWakeup(TSearchEventsProcessor::StartQueuesListingTag));
+ Parent->Server->GetRuntime()->Send(handle);
+ }
+ Parent->Server->GetRuntime()->DispatchEvents(DispatchOpts, TDuration::Minutes(1));
+ }
+ };
+ void SetUp() override {
+ TClient client(Server->GetSettings());
client.InitRootScheme();
- client.MkDir("/Root", "SQS");
- }
+ client.MkDir("/Root", "SQS");
+ }
-private:
- THolder<TDriver> Driver;
+private:
+ THolder<TDriver> Driver;
TSimpleSharedPtr<NYdb::NTable::TTableClient> TableClient;
- THolder<TServer> Server;
- TString Root = "/Root/SQS";
-
- UNIT_TEST_SUITE(TIndexProcesorTests)
- UNIT_TEST(TestCreateIndexProcessor)
- UNIT_TEST(TestSingleCreateQueueEvent)
- UNIT_TEST(TestReindexSingleQueue)
- UNIT_TEST(TestDeletedQueueNotReindexed)
- UNIT_TEST(TestManyMessages)
- UNIT_TEST(TestOver1000Queues)
- UNIT_TEST_SUITE_END();
-
- void CheckEventsLine(
- const TString& line, EEvType type, const TString& queueName = TString())
- {
- Cerr << "===CheckEventsLine: " << line << Endl;
- NJson::TJsonValue json;
- NJson::ReadJsonTree(line, &json, true);
- auto &map = json.GetMap();
- if (!queueName.empty()) {
- auto resId = map.find("resource_id")->second.GetString();
- UNIT_ASSERT_STRINGS_EQUAL(resId, queueName);
- }
- bool hasDeleted = map.find("deleted") != map.end();
- bool hasReindex = map.find("reindex_timestamp") != map.end();
- auto& res_path = map.find("resource_path")->second.GetArray();
- UNIT_ASSERT_VALUES_EQUAL(res_path.size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(res_path[0].GetMap().size(), 2);
-
-
- switch (type) {
- case EEvType::Existed:
- UNIT_ASSERT(!hasDeleted);
- UNIT_ASSERT(hasReindex);
- UNIT_ASSERT(!map.find("reindex_timestamp")->second.GetString().empty());
- break;
- case EEvType::Created:
- UNIT_ASSERT(!hasDeleted);
- UNIT_ASSERT(!hasReindex);
- break;
- case EEvType::Deleted:
- UNIT_ASSERT(hasDeleted);
- UNIT_ASSERT(!map.find("deleted")->second.GetString().empty());
- UNIT_ASSERT(!hasReindex);
- break;
- }
- }
-
- void TestCreateIndexProcessor() {
- TTestRunner("CreateIndexProcessor", this);
- }
-
- void TestSingleCreateQueueEvent() {
- TTestRunner runner{"SingleCreateQueueEvent", this};
- runner.AddEvent( "cloud1", "queue1", EEvType::Created);
- runner.DispatchEvents();
- auto messages = runner.EventsWriter->GetMessages();
- UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); // Events, reindex
- CheckEventsLine(messages[0], EEvType::Created);
- CheckEventsLine(messages[1], EEvType::Existed);
- UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0);
- }
-
- void TestReindexSingleQueue() {
- TTestRunner runner{"ReindexSingleQueue", this};
- runner.AddQueue("cloud1", "queue1");
- runner.DispatchEvents();
- auto messages = runner.EventsWriter->GetMessages();
- UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1);
- CheckEventsLine(messages[0], EEvType::Existed);
- }
-
- void TestDeletedQueueNotReindexed() {
- TTestRunner runner{"DeletedQueueNotReindexed", this};
- runner.AddQueue("cloud1", "queue2", runner.PrevTs);
- runner.AddEvent("cloud1", "queue2", EEvType::Deleted);
- Sleep(TDuration::Seconds(1));
- runner.DispatchEvents();
- auto messages = runner.EventsWriter->GetMessages();
- UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1);
- CheckEventsLine(messages[0], EEvType::Deleted);
- }
-
- void TestManyMessages() {
- TTestRunner runner{"TestManyMessages", this};
- runner.AddQueue("cloud1", "existing1", runner.PrevTs);
- runner.AddQueue("cloud1", "existing2", runner.PrevTs);
- runner.AddQueue("cloud1", "existing3", runner.PrevTs);
- runner.AddQueue("cloud1", "deleting1", runner.PrevTs);
- runner.AddQueue("cloud1", "deleting2", runner.PrevTs);
- runner.AddEvent("cloud1", "deleting1", EEvType::Deleted);
- runner.AddEvent("cloud1", "deleting2", EEvType::Deleted);
- runner.AddEvent("cloud1", "creating1", EEvType::Created);
- runner.AddEvent("cloud1", "creating2", EEvType::Created);
- UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 4);
-
- runner.DispatchEvents();
- auto messages = runner.EventsWriter->GetMessages();
- UNIT_ASSERT_VALUES_EQUAL(messages.size(), 9); //4 events, 5 queues in reindex
- for (auto i = 0u; i < 4; i++) {
- if (messages[i].find("creating") != TString::npos)
- CheckEventsLine(messages[i], EEvType::Created);
- else {
- UNIT_ASSERT(messages[i].find("deleting") != TString::npos);
- UNIT_ASSERT(messages[i].find("existing") == TString::npos);
- CheckEventsLine(messages[i], EEvType::Deleted);
- }
- }
- for (auto i = 4u; i < messages.size(); i++) {
- UNIT_ASSERT_C(
- messages[i].find("existing") != TString::npos
- || messages[i].find("creating") != TString::npos,
- messages[i].c_str()
- );
- CheckEventsLine(messages[i], EEvType::Existed);
- }
- UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0);
- }
-
- void TestOver1000Queues() {
- TTestRunner runner{"TestOver1000Queues", this};
- ui64 queuesCount = 2010;
- runner.AddQueuesBatch("cloud2", "queue-", queuesCount);
- UNIT_ASSERT_VALUES_EQUAL(runner.CountQueues(), queuesCount);
- Sleep(TDuration::Seconds(1));
- runner.EventsWriter->GetMessages(); //reset messages;
- runner.DispatchEvents();
- auto messages = runner.EventsWriter->GetMessages();
- UNIT_ASSERT_VALUES_EQUAL(messages.size(), queuesCount);
-
- }
-};
-UNIT_TEST_SUITE_REGISTRATION(TIndexProcesorTests);
-} // NKikimr::NSQS
+ THolder<TServer> Server;
+ TString Root = "/Root/SQS";
+
+ UNIT_TEST_SUITE(TIndexProcesorTests)
+ UNIT_TEST(TestCreateIndexProcessor)
+ UNIT_TEST(TestSingleCreateQueueEvent)
+ UNIT_TEST(TestReindexSingleQueue)
+ UNIT_TEST(TestDeletedQueueNotReindexed)
+ UNIT_TEST(TestManyMessages)
+ UNIT_TEST(TestOver1000Queues)
+ UNIT_TEST_SUITE_END();
+
+ void CheckEventsLine(
+ const TString& line, EEvType type, const TString& queueName = TString())
+ {
+ Cerr << "===CheckEventsLine: " << line << Endl;
+ NJson::TJsonValue json;
+ NJson::ReadJsonTree(line, &json, true);
+ auto &map = json.GetMap();
+ if (!queueName.empty()) {
+ auto resId = map.find("resource_id")->second.GetString();
+ UNIT_ASSERT_STRINGS_EQUAL(resId, queueName);
+ }
+ bool hasDeleted = map.find("deleted") != map.end();
+ bool hasReindex = map.find("reindex_timestamp") != map.end();
+ auto& res_path = map.find("resource_path")->second.GetArray();
+ UNIT_ASSERT_VALUES_EQUAL(res_path.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(res_path[0].GetMap().size(), 2);
+
+
+ switch (type) {
+ case EEvType::Existed:
+ UNIT_ASSERT(!hasDeleted);
+ UNIT_ASSERT(hasReindex);
+ UNIT_ASSERT(!map.find("reindex_timestamp")->second.GetString().empty());
+ break;
+ case EEvType::Created:
+ UNIT_ASSERT(!hasDeleted);
+ UNIT_ASSERT(!hasReindex);
+ break;
+ case EEvType::Deleted:
+ UNIT_ASSERT(hasDeleted);
+ UNIT_ASSERT(!map.find("deleted")->second.GetString().empty());
+ UNIT_ASSERT(!hasReindex);
+ break;
+ }
+ }
+
+ void TestCreateIndexProcessor() {
+ TTestRunner("CreateIndexProcessor", this);
+ }
+
+ void TestSingleCreateQueueEvent() {
+ TTestRunner runner{"SingleCreateQueueEvent", this};
+ runner.AddEvent( "cloud1", "queue1", EEvType::Created);
+ runner.DispatchEvents();
+ auto messages = runner.EventsWriter->GetMessages();
+ UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); // Events, reindex
+ CheckEventsLine(messages[0], EEvType::Created);
+ CheckEventsLine(messages[1], EEvType::Existed);
+ UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0);
+ }
+
+ void TestReindexSingleQueue() {
+ TTestRunner runner{"ReindexSingleQueue", this};
+ runner.AddQueue("cloud1", "queue1");
+ runner.DispatchEvents();
+ auto messages = runner.EventsWriter->GetMessages();
+ UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1);
+ CheckEventsLine(messages[0], EEvType::Existed);
+ }
+
+ void TestDeletedQueueNotReindexed() {
+ TTestRunner runner{"DeletedQueueNotReindexed", this};
+ runner.AddQueue("cloud1", "queue2", runner.PrevTs);
+ runner.AddEvent("cloud1", "queue2", EEvType::Deleted);
+ Sleep(TDuration::Seconds(1));
+ runner.DispatchEvents();
+ auto messages = runner.EventsWriter->GetMessages();
+ UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1);
+ CheckEventsLine(messages[0], EEvType::Deleted);
+ }
+
+ void TestManyMessages() {
+ TTestRunner runner{"TestManyMessages", this};
+ runner.AddQueue("cloud1", "existing1", runner.PrevTs);
+ runner.AddQueue("cloud1", "existing2", runner.PrevTs);
+ runner.AddQueue("cloud1", "existing3", runner.PrevTs);
+ runner.AddQueue("cloud1", "deleting1", runner.PrevTs);
+ runner.AddQueue("cloud1", "deleting2", runner.PrevTs);
+ runner.AddEvent("cloud1", "deleting1", EEvType::Deleted);
+ runner.AddEvent("cloud1", "deleting2", EEvType::Deleted);
+ runner.AddEvent("cloud1", "creating1", EEvType::Created);
+ runner.AddEvent("cloud1", "creating2", EEvType::Created);
+ UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 4);
+
+ runner.DispatchEvents();
+ auto messages = runner.EventsWriter->GetMessages();
+ UNIT_ASSERT_VALUES_EQUAL(messages.size(), 9); //4 events, 5 queues in reindex
+ for (auto i = 0u; i < 4; i++) {
+ if (messages[i].find("creating") != TString::npos)
+ CheckEventsLine(messages[i], EEvType::Created);
+ else {
+ UNIT_ASSERT(messages[i].find("deleting") != TString::npos);
+ UNIT_ASSERT(messages[i].find("existing") == TString::npos);
+ CheckEventsLine(messages[i], EEvType::Deleted);
+ }
+ }
+ for (auto i = 4u; i < messages.size(); i++) {
+ UNIT_ASSERT_C(
+ messages[i].find("existing") != TString::npos
+ || messages[i].find("creating") != TString::npos,
+ messages[i].c_str()
+ );
+ CheckEventsLine(messages[i], EEvType::Existed);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(runner.CountEvents(), 0);
+ }
+
+ void TestOver1000Queues() {
+ TTestRunner runner{"TestOver1000Queues", this};
+ ui64 queuesCount = 2010;
+ runner.AddQueuesBatch("cloud2", "queue-", queuesCount);
+ UNIT_ASSERT_VALUES_EQUAL(runner.CountQueues(), queuesCount);
+ Sleep(TDuration::Seconds(1));
+ runner.EventsWriter->GetMessages(); //reset messages;
+ runner.DispatchEvents();
+ auto messages = runner.EventsWriter->GetMessages();
+ UNIT_ASSERT_VALUES_EQUAL(messages.size(), queuesCount);
+
+ }
+};
+UNIT_TEST_SUITE_REGISTRATION(TIndexProcesorTests);
+} // NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp
index f643cce14b..21c0f887ba 100644
--- a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp
+++ b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.cpp
@@ -1,20 +1,20 @@
-#include "test_events_writer.h"
-
-namespace NKikimr::NSQS {
-
-void TTestEventsWriter::Write(const TString& data) {
- with_lock(Lock) {
- Messages.emplace_back(data);
- }
-}
-
-TVector<TString> TTestEventsWriter::GetMessages() {
- TVector<TString> result;
- with_lock(Lock) {
- result = std::move(Messages);
- Messages.clear();
- }
- return result;
-}
-
-} //namespace NKikimr::NSQS
+#include "test_events_writer.h"
+
+namespace NKikimr::NSQS {
+
+void TTestEventsWriter::Write(const TString& data) {
+ with_lock(Lock) {
+ Messages.emplace_back(data);
+ }
+}
+
+TVector<TString> TTestEventsWriter::GetMessages() {
+ TVector<TString> result;
+ with_lock(Lock) {
+ result = std::move(Messages);
+ Messages.clear();
+ }
+ return result;
+}
+
+} //namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h
index 3364007312..4ff876074f 100644
--- a/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h
+++ b/ydb/core/ymq/actor/yc_search_ut/test_events_writer.h
@@ -1,22 +1,22 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/ymq/actor/index_events_processor.h>
-
-namespace NKikimr::NSQS {
-
-class TTestEventsWriter : public IEventsWriterWrapper {
-public:
- TTestEventsWriter() = default;
- void Write(const TString& data) override;
- TVector<TString> GetMessages();
-
- void CloseImpl() override {
- }
-
-private:
- TAdaptiveLock Lock;
- TVector<TString> Messages;
-};
-using TMockSessionPtr = TIntrusivePtr<TTestEventsWriter>;
-
-} //namespace NKikimr::NSQS
+
+namespace NKikimr::NSQS {
+
+class TTestEventsWriter : public IEventsWriterWrapper {
+public:
+ TTestEventsWriter() = default;
+ void Write(const TString& data) override;
+ TVector<TString> GetMessages();
+
+ void CloseImpl() override {
+ }
+
+private:
+ TAdaptiveLock Lock;
+ TVector<TString> Messages;
+};
+using TMockSessionPtr = TIntrusivePtr<TTestEventsWriter>;
+
+} //namespace NKikimr::NSQS
diff --git a/ydb/core/ymq/actor/yc_search_ut/ya.make b/ydb/core/ymq/actor/yc_search_ut/ya.make
index 7465570487..19d200d2e2 100644
--- a/ydb/core/ymq/actor/yc_search_ut/ya.make
+++ b/ydb/core/ymq/actor/yc_search_ut/ya.make
@@ -1,25 +1,25 @@
-OWNER(
- komels
+OWNER(
+ komels
g:kikimr
- g:sqs
-)
-
-UNITTEST()
-
-PEERDIR(
+ g:sqs
+)
+
+UNITTEST()
+
+PEERDIR(
ydb/core/testlib
ydb/core/ymq/actor
-)
-
-SRCS(
- index_events_processor_ut.cpp
- test_events_writer.cpp
-)
+)
+
+SRCS(
+ index_events_processor_ut.cpp
+ test_events_writer.cpp
+)
+
+SIZE(MEDIUM)
-SIZE(MEDIUM)
+TIMEOUT(300)
-TIMEOUT(300)
+YQL_LAST_ABI_VERSION()
-YQL_LAST_ABI_VERSION()
-
-END()
+END()
diff --git a/ydb/core/ymq/base/action.cpp b/ydb/core/ymq/base/action.cpp
index b49c145146..b601d72d87 100644
--- a/ydb/core/ymq/base/action.cpp
+++ b/ydb/core/ymq/base/action.cpp
@@ -14,43 +14,43 @@ constexpr ui32 FOR_MESSAGE = 4;
constexpr ui32 BATCH = 8;
constexpr ui32 FAST = 16;
constexpr ui32 PRIVATE = 32;
-constexpr ui32 YMQ_FOR_QUEUE = 64;
-constexpr ui32 YMQ_FOR_USER = 128;
+constexpr ui32 YMQ_FOR_QUEUE = 64;
+constexpr ui32 YMQ_FOR_USER = 128;
struct TActionProps {
TString StringName;
- TString ConvMethodName;
+ TString ConvMethodName;
EAction Action;
ui32 Flags;
EAction NonBatchAction;
};
static const TActionProps ActionProps[] = {
- {"Unknown", "unknown", EAction::Unknown, 0, EAction::Unknown},
- {"ChangeMessageVisibility", "change_message_visibility", EAction::ChangeMessageVisibility, FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ChangeMessageVisibility},
- {"ChangeMessageVisibilityBatch", "change_message_visibility_batch", EAction::ChangeMessageVisibilityBatch, FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::ChangeMessageVisibility},
- {"CreateQueue", "create_queue", EAction::CreateQueue, FOR_USER | YMQ_FOR_USER, EAction::CreateQueue},
- {"CreateUser", "create_user", EAction::CreateUser, FOR_USER, EAction::CreateUser},
- {"GetQueueAttributes", "get_queue_attributes", EAction::GetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueAttributes},
- {"GetQueueAttributesBatch", "get_queue_attributes_batch", EAction::GetQueueAttributesBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::GetQueueAttributes},
- {"GetQueueUrl", "get_queue_url", EAction::GetQueueUrl, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueUrl},
- {"DeleteMessage", "delete_message", EAction::DeleteMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::DeleteMessage},
- {"DeleteMessageBatch", "delete_message_batch", EAction::DeleteMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::DeleteMessage},
- {"DeleteQueue", "delete_queue", EAction::DeleteQueue, FOR_USER | YMQ_FOR_USER, EAction::DeleteQueue},
- {"DeleteQueueBatch", "delete_queue_batch", EAction::DeleteQueueBatch, FOR_USER | BATCH | PRIVATE, EAction::DeleteQueue},
- {"DeleteUser", "delete_user", EAction::DeleteUser, FOR_USER, EAction::DeleteUser},
- {"ListQueues", "list_queues", EAction::ListQueues, FOR_USER | YMQ_FOR_USER | FAST, EAction::ListQueues},
- {"ListUsers", "list_users", EAction::ListUsers, FOR_USER | FAST, EAction::ListUsers},
- {"PurgeQueue", "purge_queue", EAction::PurgeQueue, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::PurgeQueue},
- {"PurgeQueueBatch", "purge_queue_batch", EAction::PurgeQueueBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::PurgeQueue},
- {"ReceiveMessage", "receive_message", EAction::ReceiveMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ReceiveMessage},
- {"SendMessage", "send_message", EAction::SendMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::SendMessage},
- {"SendMessageBatch", "send_message_batch", EAction::SendMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::SendMessage},
- {"SetQueueAttributes", "set_queue_attributes", EAction::SetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::SetQueueAttributes},
- {"ModifyPermissions", "modify_permissions", EAction::ModifyPermissions, FOR_USER | FAST, EAction::ModifyPermissions},
- {"ListPermissions", "list_permissions", EAction::ListPermissions, FOR_USER | FAST, EAction::ListPermissions},
- {"ListDeadLetterSourceQueues", "list_dead_letter_source_queues", EAction::ListDeadLetterSourceQueues, FOR_QUEUE | FAST, EAction::ListDeadLetterSourceQueues},
- {"CountQueues", "count_queues", EAction::CountQueues, FOR_USER | FAST | PRIVATE, EAction::CountQueues},
+ {"Unknown", "unknown", EAction::Unknown, 0, EAction::Unknown},
+ {"ChangeMessageVisibility", "change_message_visibility", EAction::ChangeMessageVisibility, FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ChangeMessageVisibility},
+ {"ChangeMessageVisibilityBatch", "change_message_visibility_batch", EAction::ChangeMessageVisibilityBatch, FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::ChangeMessageVisibility},
+ {"CreateQueue", "create_queue", EAction::CreateQueue, FOR_USER | YMQ_FOR_USER, EAction::CreateQueue},
+ {"CreateUser", "create_user", EAction::CreateUser, FOR_USER, EAction::CreateUser},
+ {"GetQueueAttributes", "get_queue_attributes", EAction::GetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueAttributes},
+ {"GetQueueAttributesBatch", "get_queue_attributes_batch", EAction::GetQueueAttributesBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::GetQueueAttributes},
+ {"GetQueueUrl", "get_queue_url", EAction::GetQueueUrl, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::GetQueueUrl},
+ {"DeleteMessage", "delete_message", EAction::DeleteMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::DeleteMessage},
+ {"DeleteMessageBatch", "delete_message_batch", EAction::DeleteMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::DeleteMessage},
+ {"DeleteQueue", "delete_queue", EAction::DeleteQueue, FOR_USER | YMQ_FOR_USER, EAction::DeleteQueue},
+ {"DeleteQueueBatch", "delete_queue_batch", EAction::DeleteQueueBatch, FOR_USER | BATCH | PRIVATE, EAction::DeleteQueue},
+ {"DeleteUser", "delete_user", EAction::DeleteUser, FOR_USER, EAction::DeleteUser},
+ {"ListQueues", "list_queues", EAction::ListQueues, FOR_USER | YMQ_FOR_USER | FAST, EAction::ListQueues},
+ {"ListUsers", "list_users", EAction::ListUsers, FOR_USER | FAST, EAction::ListUsers},
+ {"PurgeQueue", "purge_queue", EAction::PurgeQueue, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::PurgeQueue},
+ {"PurgeQueueBatch", "purge_queue_batch", EAction::PurgeQueueBatch, FOR_USER | BATCH | FAST | PRIVATE, EAction::PurgeQueue},
+ {"ReceiveMessage", "receive_message", EAction::ReceiveMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::ReceiveMessage},
+ {"SendMessage", "send_message", EAction::SendMessage, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | FAST, EAction::SendMessage},
+ {"SendMessageBatch", "send_message_batch", EAction::SendMessageBatch, FOR_QUEUE | YMQ_FOR_QUEUE | FOR_MESSAGE | BATCH | FAST, EAction::SendMessage},
+ {"SetQueueAttributes", "set_queue_attributes", EAction::SetQueueAttributes, FOR_QUEUE | YMQ_FOR_QUEUE | FAST, EAction::SetQueueAttributes},
+ {"ModifyPermissions", "modify_permissions", EAction::ModifyPermissions, FOR_USER | FAST, EAction::ModifyPermissions},
+ {"ListPermissions", "list_permissions", EAction::ListPermissions, FOR_USER | FAST, EAction::ListPermissions},
+ {"ListDeadLetterSourceQueues", "list_dead_letter_source_queues", EAction::ListDeadLetterSourceQueues, FOR_QUEUE | FAST, EAction::ListDeadLetterSourceQueues},
+ {"CountQueues", "count_queues", EAction::CountQueues, FOR_USER | FAST | PRIVATE, EAction::CountQueues},
};
static_assert(Y_ARRAY_SIZE(ActionProps) == EAction::ActionsArraySize);
@@ -88,10 +88,10 @@ const TString& ActionToString(EAction action) {
return GetProps(action).StringName;
}
-const TString& ActionToCloudConvMethod(EAction action) {
- return GetProps(action).ConvMethodName;
-}
-
+const TString& ActionToCloudConvMethod(EAction action) {
+ return GetProps(action).ConvMethodName;
+}
+
bool IsBatchAction(EAction action) {
return GetProps(action).Flags & BATCH;
}
@@ -104,18 +104,18 @@ bool IsActionForQueue(EAction action) {
return GetProps(action).Flags & FOR_QUEUE;
}
-bool IsActionForQueueYMQ(EAction action) {
- return GetProps(action).Flags & YMQ_FOR_QUEUE;
-}
-
+bool IsActionForQueueYMQ(EAction action) {
+ return GetProps(action).Flags & YMQ_FOR_QUEUE;
+}
+
bool IsActionForUser(EAction action) {
return GetProps(action).Flags & FOR_USER;
}
-bool IsActionForUserYMQ(EAction action) {
- return GetProps(action).Flags & YMQ_FOR_USER;
-}
-
+bool IsActionForUserYMQ(EAction action) {
+ return GetProps(action).Flags & YMQ_FOR_USER;
+}
+
bool IsActionForMessage(EAction action) {
return GetProps(action).Flags & FOR_MESSAGE;
}
diff --git a/ydb/core/ymq/base/action.h b/ydb/core/ymq/base/action.h
index c7e87c7203..ede7922dd1 100644
--- a/ydb/core/ymq/base/action.h
+++ b/ydb/core/ymq/base/action.h
@@ -36,13 +36,13 @@ enum EAction {
EAction ActionFromString(const TString& name);
const TString& ActionToString(EAction action);
-const TString& ActionToCloudConvMethod(EAction action);
+const TString& ActionToCloudConvMethod(EAction action);
bool IsBatchAction(EAction action);
bool IsActionForQueue(EAction action);
-bool IsActionForQueueYMQ(EAction action);
+bool IsActionForQueueYMQ(EAction action);
bool IsActionForUser(EAction action);
-bool IsActionForUserYMQ(EAction action);
+bool IsActionForUserYMQ(EAction action);
bool IsProxyAction(EAction action);
bool IsActionForMessage(EAction action);
bool IsFastAction(EAction action);
diff --git a/ydb/core/ymq/base/counters.cpp b/ydb/core/ymq/base/counters.cpp
index b8a77d62fb..90a9f5000f 100644
--- a/ydb/core/ymq/base/counters.cpp
+++ b/ydb/core/ymq/base/counters.cpp
@@ -14,12 +14,12 @@
namespace NKikimr::NSQS {
extern const TString DEFAULT_COUNTER_NAME = "sensor";
-extern const TString DEFAULT_YMQ_COUNTER_NAME = "name";
-extern const TString ACTION_CNTR_PREFIX = "api.http.";
-extern const TString QUEUE_CNTR_PREFIX = "queue.messages.";
-extern const TString METHOD_LABLE = "method";
+extern const TString DEFAULT_YMQ_COUNTER_NAME = "name";
+extern const TString ACTION_CNTR_PREFIX = "api.http.";
+extern const TString QUEUE_CNTR_PREFIX = "queue.messages.";
+extern const TString METHOD_LABLE = "method";
static const TString USER_LABEL = "user";
-static const TString CLOUD_LABEL = "cloud";
+static const TString CLOUD_LABEL = "cloud";
static const TString FOLDER_LABEL = "folder";
static const TString QUEUE_LABEL = "queue";
extern const TString TOTAL_COUNTER_LABEL = "total";
@@ -40,21 +40,21 @@ static const NMonitoring::TBucketBounds FastActionsDurationBucketsMs = {
5000,
};
-static const NMonitoring::TBucketBounds YmqFastActionsDurationBucketsMs = {
- 5,
- 10,
- 25,
- 50,
- 75,
- 100,
- 150,
- 300,
- 500,
- 1000,
- 5000,
-};
-
-
+static const NMonitoring::TBucketBounds YmqFastActionsDurationBucketsMs = {
+ 5,
+ 10,
+ 25,
+ 50,
+ 75,
+ 100,
+ 150,
+ 300,
+ 500,
+ 1000,
+ 5000,
+};
+
+
static const NMonitoring::TBucketBounds SlowActionsDurationBucketsMs = {
100,
150,
@@ -64,18 +64,18 @@ static const NMonitoring::TBucketBounds SlowActionsDurationBucketsMs = {
5000,
};
-static const NMonitoring::TBucketBounds YmqSlowActionsDurationBucketsMs = {
- 100,
- 150,
- 300,
- 500,
- 1000,
- 5000,
- 10'000,
- 20'000,
- 50'000
-};
-
+static const NMonitoring::TBucketBounds YmqSlowActionsDurationBucketsMs = {
+ 100,
+ 150,
+ 300,
+ 500,
+ 1000,
+ 5000,
+ 10'000,
+ 20'000,
+ 50'000
+};
+
static const NMonitoring::TBucketBounds DurationBucketsMs = {
5,
10,
@@ -118,22 +118,22 @@ static const NMonitoring::TBucketBounds ClientDurationBucketsMs = {
25'000'000,
50'000'000,
};
-static const NMonitoring::TBucketBounds YmqClientDurationBucketsMs = {
- 100,
- 200,
- 500,
- 1'000,
- 2'000,
- 5'000,
- 10'000,
- 20'000,
- 60'000,
- 120'000,
- 300'000,
- 600'000,
- 3'600'000,
- 7'200'000,
-};
+static const NMonitoring::TBucketBounds YmqClientDurationBucketsMs = {
+ 100,
+ 200,
+ 500,
+ 1'000,
+ 2'000,
+ 5'000,
+ 10'000,
+ 20'000,
+ 60'000,
+ 120'000,
+ 300'000,
+ 600'000,
+ 3'600'000,
+ 7'200'000,
+};
static const NMonitoring::TBucketBounds GetQuotaDurationBucketsMs = {
1,
@@ -301,46 +301,46 @@ static_assert(AbsDiffLessThanCounter(HttpUserCountersDescriptor.SizeOfCounters()
TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup) {
return GetServiceCounters(countersRoot, "sqs")->GetSubgroup("subsystem", subgroup);
}
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot) {
- // Remove subgroup and don't have subsystem (is this correct - ?)
- return GetServiceCounters(countersRoot, "ymq_public");
-}
-
-static TIntrusivePtrCntrCouple GetUserCounters(
- const TIntrusivePtrCntrCouple & sqsCoreCounters, const TString& userName
-) {
- return {
- sqsCoreCounters.SqsCounters->GetSubgroup(USER_LABEL, userName),
- sqsCoreCounters.YmqCounters ? sqsCoreCounters.YmqCounters->GetSubgroup(CLOUD_LABEL, userName) : nullptr
- };
-}
-
-TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) {
- return {
- userCounters.SqsCounters->GetSubgroup(FOLDER_LABEL, folderId),
- userCounters.YmqCounters ? userCounters.YmqCounters->GetSubgroup(FOLDER_LABEL, folderId) : nullptr
- };
-}
-
-void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) {
- if (userCounters.YmqCounters) {
- userCounters.YmqCounters->RemoveSubgroup(FOLDER_LABEL, folderId);
- }
-}
-
-static TIntrusivePtrCntrCouple GetQueueCounters(const TIntrusivePtrCntrCouple & userOrFolderCounters, const TString& queueName) {
- return {
- userOrFolderCounters.SqsCounters->GetSubgroup(QUEUE_LABEL, queueName),
- userOrFolderCounters.YmqCounters ? userOrFolderCounters.YmqCounters->GetSubgroup(QUEUE_LABEL, queueName) : nullptr
- };
-}
-
-std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters(
- const TIntrusivePtrCntrCouple& sqsCounters, const TQueuePath& queuePath
-) {
- TIntrusivePtrCntrCouple userCounters;
- TIntrusivePtrCntrCouple queueCounters;
- if (queuePath.UserName && sqsCounters.SqsCounters) {
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot) {
+ // Remove subgroup and don't have subsystem (is this correct - ?)
+ return GetServiceCounters(countersRoot, "ymq_public");
+}
+
+static TIntrusivePtrCntrCouple GetUserCounters(
+ const TIntrusivePtrCntrCouple & sqsCoreCounters, const TString& userName
+) {
+ return {
+ sqsCoreCounters.SqsCounters->GetSubgroup(USER_LABEL, userName),
+ sqsCoreCounters.YmqCounters ? sqsCoreCounters.YmqCounters->GetSubgroup(CLOUD_LABEL, userName) : nullptr
+ };
+}
+
+TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) {
+ return {
+ userCounters.SqsCounters->GetSubgroup(FOLDER_LABEL, folderId),
+ userCounters.YmqCounters ? userCounters.YmqCounters->GetSubgroup(FOLDER_LABEL, folderId) : nullptr
+ };
+}
+
+void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId) {
+ if (userCounters.YmqCounters) {
+ userCounters.YmqCounters->RemoveSubgroup(FOLDER_LABEL, folderId);
+ }
+}
+
+static TIntrusivePtrCntrCouple GetQueueCounters(const TIntrusivePtrCntrCouple & userOrFolderCounters, const TString& queueName) {
+ return {
+ userOrFolderCounters.SqsCounters->GetSubgroup(QUEUE_LABEL, queueName),
+ userOrFolderCounters.YmqCounters ? userOrFolderCounters.YmqCounters->GetSubgroup(QUEUE_LABEL, queueName) : nullptr
+ };
+}
+
+std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters(
+ const TIntrusivePtrCntrCouple& sqsCounters, const TQueuePath& queuePath
+) {
+ TIntrusivePtrCntrCouple userCounters;
+ TIntrusivePtrCntrCouple queueCounters;
+ if (queuePath.UserName && sqsCounters.SqsCounters) {
userCounters = GetUserCounters(sqsCounters, queuePath.UserName);
if (queuePath.QueueName) {
queueCounters = GetQueueCounters(userCounters, queuePath.QueueName);
@@ -349,19 +349,19 @@ std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCount
return { std::move(userCounters), std::move(queueCounters) };
}
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters(
- const TIntrusivePtrCntrCouple& rootCounters, const NKikimrConfig::TSqsConfig& cfg
-) {
- return GetAggregatedCountersFromUserCounters(GetUserCounters(rootCounters, TOTAL_COUNTER_LABEL), cfg);
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters(
+ const TIntrusivePtrCntrCouple& rootCounters, const NKikimrConfig::TSqsConfig& cfg
+) {
+ return GetAggregatedCountersFromUserCounters(GetUserCounters(rootCounters, TOTAL_COUNTER_LABEL), cfg);
}
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters(
- const TIntrusivePtrCntrCouple& userCounters, const NKikimrConfig::TSqsConfig& cfg
-) {
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters(
+ const TIntrusivePtrCntrCouple& userCounters, const NKikimrConfig::TSqsConfig& cfg
+) {
if (cfg.GetYandexCloudMode()) {
- return GetQueueCounters(GetFolderCounters(userCounters, TOTAL_COUNTER_LABEL), TOTAL_COUNTER_LABEL).SqsCounters;
+ return GetQueueCounters(GetFolderCounters(userCounters, TOTAL_COUNTER_LABEL), TOTAL_COUNTER_LABEL).SqsCounters;
} else {
- return GetQueueCounters(userCounters, TOTAL_COUNTER_LABEL).SqsCounters;
+ return GetQueueCounters(userCounters, TOTAL_COUNTER_LABEL).SqsCounters;
}
}
@@ -371,33 +371,33 @@ ELaziness Lazy(const NKikimrConfig::TSqsConfig& cfg) {
#define INIT_COUNTER_WITH_NAME(rootCounters, variable, name, expiring, valueType, lazy) \
variable.Init(rootCounters, expiring, valueType, name, lazy)
-#define INIT_COUNTER_WITH_NAME_AND_LABEL(rootCounters, variable, labelName, name, expiring, valueType, lazy) \
- variable.Init(rootCounters, expiring, valueType, labelName, name, lazy)
-
-#define INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, sqsName, ymqName, expiring, valueType, lazy, aggr) \
- sqsCounter.Init(rootCounters.SqsCounters, expiring, valueType, sqsName, lazy); \
- if (rootCounters.YmqCounters && !aggr) { \
- ymqCounter.Init(rootCounters.YmqCounters, expiring, valueType, DEFAULT_YMQ_COUNTER_NAME, ymqName, ELaziness::OnStart); \
- }
-
+#define INIT_COUNTER_WITH_NAME_AND_LABEL(rootCounters, variable, labelName, name, expiring, valueType, lazy) \
+ variable.Init(rootCounters, expiring, valueType, labelName, name, lazy)
+
+#define INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, sqsName, ymqName, expiring, valueType, lazy, aggr) \
+ sqsCounter.Init(rootCounters.SqsCounters, expiring, valueType, sqsName, lazy); \
+ if (rootCounters.YmqCounters && !aggr) { \
+ ymqCounter.Init(rootCounters.YmqCounters, expiring, valueType, DEFAULT_YMQ_COUNTER_NAME, ymqName, ELaziness::OnStart); \
+ }
+
#define INIT_COUNTER(rootCounters, variable, expiring, valueType, lazy) \
INIT_COUNTER_WITH_NAME(rootCounters, variable, Y_STRINGIZE(variable), expiring, valueType, lazy)
-
-#define INIT_COUNTERS_COUPLE(rootCounters, sqsCounter, ymqCounter, expiring, valueType, lazy, aggr) \
- INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, Y_STRINGIZE(sqsCounter), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqCounter), expiring, valueType, lazy, aggr)
-
-#define INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, sqsName, ymqName, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \
- sqsHistogram.Init(rootCounters.SqsCounters, expiring, sqsBuckets, sqsName, lazy); \
- if (rootCounters.YmqCounters && !aggr) { \
- ymqHistogram.Init(rootCounters.YmqCounters, expiring, ymqBuckets, DEFAULT_YMQ_COUNTER_NAME, ymqName, lazy); \
- }
-
-#define INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \
- INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, Y_STRINGIZE(sqsHistogram), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqHistogram), expiring, sqsBuckets, ymqBuckets, lazy, aggr)
-
-#define INIT_HISTOGRAMS_COUPLE(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, lazy, aggr) \
- INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, Y_CAT(Ymq, buckets), lazy, aggr)
-
+
+#define INIT_COUNTERS_COUPLE(rootCounters, sqsCounter, ymqCounter, expiring, valueType, lazy, aggr) \
+ INIT_COUNTERS_COUPLE_WITH_NAMES(rootCounters, sqsCounter, ymqCounter, Y_STRINGIZE(sqsCounter), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqCounter), expiring, valueType, lazy, aggr)
+
+#define INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, sqsName, ymqName, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \
+ sqsHistogram.Init(rootCounters.SqsCounters, expiring, sqsBuckets, sqsName, lazy); \
+ if (rootCounters.YmqCounters && !aggr) { \
+ ymqHistogram.Init(rootCounters.YmqCounters, expiring, ymqBuckets, DEFAULT_YMQ_COUNTER_NAME, ymqName, lazy); \
+ }
+
+#define INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, sqsBuckets, ymqBuckets, lazy, aggr) \
+ INIT_HISTOGRAMS_COUPLE_WITH_NAMES(rootCounters, sqsHistogram, ymqHistogram, Y_STRINGIZE(sqsHistogram), TString(QUEUE_CNTR_PREFIX) + Y_STRINGIZE(ymqHistogram), expiring, sqsBuckets, ymqBuckets, lazy, aggr)
+
+#define INIT_HISTOGRAMS_COUPLE(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, lazy, aggr) \
+ INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(rootCounters, sqsHistogram, ymqHistogram, expiring, buckets, Y_CAT(Ymq, buckets), lazy, aggr)
+
#define INIT_HISTOGRAM_COUNTER_WITH_NAME(rootCounters, variable, name, expiring, buckets, lazy) \
variable.Init(rootCounters, expiring, buckets, name, lazy)
#define INIT_HISTOGRAM_COUNTER(rootCounters, variable, expiring, buckets, lazy) \
@@ -417,28 +417,28 @@ void TActionCounters::Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusiv
}
}
-void TYmqActionCounters::Init(
- const NKikimrConfig::TSqsConfig&, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters,
- EAction action, const TString& labelName, const TString& namePrefix, ELifetime lifetime
-) {
- const auto& methodName = ActionToCloudConvMethod(action);
- SubGroup = rootCounters->GetSubgroup(labelName, methodName);
- INIT_COUNTER_WITH_NAME_AND_LABEL(
- SubGroup, Success, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "requests_count_per_second",
- lifetime,EValueType::Derivative, ELaziness::OnStart);
- INIT_COUNTER_WITH_NAME_AND_LABEL(
- SubGroup, Errors, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "errors_count_per_second",
- lifetime,EValueType::Derivative, ELaziness::OnStart
- );
- // ! - No inflight counter
-
- Duration.Init(
- SubGroup, lifetime, IsFastAction(action) ? YmqFastActionsDurationBucketsMs : YmqSlowActionsDurationBucketsMs,
- DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "request_duration_milliseconds",
- ELaziness::OnStart
- );
-}
-
+void TYmqActionCounters::Init(
+ const NKikimrConfig::TSqsConfig&, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters,
+ EAction action, const TString& labelName, const TString& namePrefix, ELifetime lifetime
+) {
+ const auto& methodName = ActionToCloudConvMethod(action);
+ SubGroup = rootCounters->GetSubgroup(labelName, methodName);
+ INIT_COUNTER_WITH_NAME_AND_LABEL(
+ SubGroup, Success, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "requests_count_per_second",
+ lifetime,EValueType::Derivative, ELaziness::OnStart);
+ INIT_COUNTER_WITH_NAME_AND_LABEL(
+ SubGroup, Errors, DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "errors_count_per_second",
+ lifetime,EValueType::Derivative, ELaziness::OnStart
+ );
+ // ! - No inflight counter
+
+ Duration.Init(
+ SubGroup, lifetime, IsFastAction(action) ? YmqFastActionsDurationBucketsMs : YmqSlowActionsDurationBucketsMs,
+ DEFAULT_YMQ_COUNTER_NAME, TStringBuilder() << namePrefix << "request_duration_milliseconds",
+ ELaziness::OnStart
+ );
+}
+
void TActionCounters::SetAggregatedParent(TActionCounters* parent) {
ActionCountersDescriptor.SetAggregatedParent(this, parent);
}
@@ -514,15 +514,15 @@ void TAPIStatusesCounters::SetAggregatedParent(TAPIStatusesCounters* parent) {
}
TQueueCounters::TQueueCounters(const NKikimrConfig::TSqsConfig& cfg,
- const TIntrusivePtrCntrCouple& rootCounters,
+ const TIntrusivePtrCntrCouple& rootCounters,
const TUserCounters* userCounters,
const TString& queueName,
const TString& folderId,
bool insertCounters,
bool aggregated)
- : RootCounters(rootCounters)
+ : RootCounters(rootCounters)
, UserCounters(userCounters->UserCounters)
- , FolderCounters(folderId ? GetFolderCounters(UserCounters, folderId) : TIntrusivePtrCntrCouple{})
+ , FolderCounters(folderId ? GetFolderCounters(UserCounters, folderId) : TIntrusivePtrCntrCouple{})
, Cfg(&cfg)
, QueueName(queueName)
, AggregatedCounters(aggregated)
@@ -530,120 +530,120 @@ TQueueCounters::TQueueCounters(const NKikimrConfig::TSqsConfig& cfg,
, AllocPoolCounters(userCounters->DetailedCounters.TransactionCounters->AllocPoolCounters)
{
if (insertCounters) {
- QueueCounters = GetQueueCounters(FolderCounters.Defined() ? FolderCounters : UserCounters, queueName);
+ QueueCounters = GetQueueCounters(FolderCounters.Defined() ? FolderCounters : UserCounters, queueName);
} else {
- QueueCounters = {new NMonitoring::TDynamicCounters(), new NMonitoring::TDynamicCounters()};
+ QueueCounters = {new NMonitoring::TDynamicCounters(), new NMonitoring::TDynamicCounters()};
}
InitCounters();
}
void TQueueCounters::InitCounters(bool forLeaderNode) {
if (!RequestTimeouts) {
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- RequestTimeouts, request_timeouts_count_per_second,
- ELifetime::Persistent, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ RequestTimeouts, request_timeouts_count_per_second,
+ ELifetime::Persistent, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
}
-
+
if (forLeaderNode) {
- INIT_COUNTER(QueueCounters.SqsCounters, RequestsThrottled, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart);
+ INIT_COUNTER(QueueCounters.SqsCounters, RequestsThrottled, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart);
- INIT_COUNTER(QueueCounters.SqsCounters, QueueMasterStartProblems, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart);
+ INIT_COUNTER(QueueCounters.SqsCounters, QueueMasterStartProblems, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart);
INIT_COUNTER(QueueCounters.SqsCounters, QueueLeaderStartProblems, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart);
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- MessagesPurged, purged_count_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- ELaziness::OnStart, AggregatedCounters
- );
-
- INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(
- QueueCounters, MessageReceiveAttempts, receive_attempts_count_rate,
- ELifetime::Expiring, MessageReceiveAttemptsBuckets, MessageReceiveAttemptsBuckets,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_HISTOGRAMS_COUPLE(
- QueueCounters,
- ClientMessageProcessing_Duration, client_processing_duration_milliseconds,
- ELifetime::Expiring, ClientDurationBucketsMs,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_HISTOGRAMS_COUPLE(
- QueueCounters,
- MessageReside_Duration, reside_duration_milliseconds,
- ELifetime::Expiring, ClientDurationBucketsMs,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- DeleteMessage_Count, deleted_count_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
-
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- ReceiveMessage_Count, received_count_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- ReceiveMessage_BytesRead, received_bytes_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
-
- INIT_COUNTER(QueueCounters.SqsCounters, MessagesMovedToDLQ, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart);
-
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- SendMessage_DeduplicationCount, deduplicated_count_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- SendMessage_Count, sent_count_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- SendMessage_BytesWritten, sent_bytes_per_second,
- ELifetime::Expiring, EValueType::Derivative,
- Lazy(*Cfg), AggregatedCounters
- );
-
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- MessagesCount, stored_count,
- ELifetime::Expiring, EValueType::Absolute,
- Lazy(*Cfg), AggregatedCounters
- );
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- InflyMessagesCount, inflight_count,
- ELifetime::Expiring, EValueType::Absolute,
- Lazy(*Cfg), AggregatedCounters
- );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ MessagesPurged, purged_count_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ ELaziness::OnStart, AggregatedCounters
+ );
+
+ INIT_HISTOGRAMS_COUPLE_WITH_BUCKETS(
+ QueueCounters, MessageReceiveAttempts, receive_attempts_count_rate,
+ ELifetime::Expiring, MessageReceiveAttemptsBuckets, MessageReceiveAttemptsBuckets,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_HISTOGRAMS_COUPLE(
+ QueueCounters,
+ ClientMessageProcessing_Duration, client_processing_duration_milliseconds,
+ ELifetime::Expiring, ClientDurationBucketsMs,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_HISTOGRAMS_COUPLE(
+ QueueCounters,
+ MessageReside_Duration, reside_duration_milliseconds,
+ ELifetime::Expiring, ClientDurationBucketsMs,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ DeleteMessage_Count, deleted_count_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
+
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ ReceiveMessage_EmptyCount, empty_receive_attempts_count_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ ReceiveMessage_Count, received_count_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ ReceiveMessage_BytesRead, received_bytes_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
+
+ INIT_COUNTER(QueueCounters.SqsCounters, MessagesMovedToDLQ, ELifetime::Expiring, EValueType::Derivative, ELaziness::OnStart);
+
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ SendMessage_DeduplicationCount, deduplicated_count_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ SendMessage_Count, sent_count_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ SendMessage_BytesWritten, sent_bytes_per_second,
+ ELifetime::Expiring, EValueType::Derivative,
+ Lazy(*Cfg), AggregatedCounters
+ );
+
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ MessagesCount, stored_count,
+ ELifetime::Expiring, EValueType::Absolute,
+ Lazy(*Cfg), AggregatedCounters
+ );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ InflyMessagesCount, inflight_count,
+ ELifetime::Expiring, EValueType::Absolute,
+ Lazy(*Cfg), AggregatedCounters
+ );
if (!AggregatedCounters) { // OldestMessageAgeSeconds will not be aggregated properly.
- INIT_COUNTERS_COUPLE(
- QueueCounters,
- OldestMessageAgeSeconds, oldest_age_milliseconds,
- ELifetime::Expiring, EValueType::Absolute,
- ELaziness::OnStart, false
- );
+ INIT_COUNTERS_COUPLE(
+ QueueCounters,
+ OldestMessageAgeSeconds, oldest_age_milliseconds,
+ ELifetime::Expiring, EValueType::Absolute,
+ ELaziness::OnStart, false
+ );
}
}
@@ -653,14 +653,14 @@ void TQueueCounters::InitCounters(bool forLeaderNode) {
SqsActionCounters[action].Init(*Cfg, QueueCounters.SqsCounters, action, forLeaderNode ? ELifetime::Expiring : ELifetime::Persistent);
}
}
- if (IsActionForQueueYMQ(action) && QueueCounters.YmqCounters && !AggregatedCounters) {
+ if (IsActionForQueueYMQ(action) && QueueCounters.YmqCounters && !AggregatedCounters) {
if (forLeaderNode && IsProxyAction(action) || !forLeaderNode && !IsProxyAction(action)) {
- YmqActionCounters[action].Init(
- *Cfg, QueueCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX,
+ YmqActionCounters[action].Init(
+ *Cfg, QueueCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX,
forLeaderNode ? ELifetime::Expiring : ELifetime::Persistent
- );
- }
- }
+ );
+ }
+ }
}
DetailedCounters.Init(QueueCounters.SqsCounters, AllocPoolCounters, forLeaderNode);
@@ -691,17 +691,17 @@ void TQueueCounters::TDetailedCounters::SetAggregatedParent(TQueueCounters::TDet
void TQueueCounters::InsertCounters() {
auto insert = [&](auto& parent) {
- if (!parent.SqsCounters->FindSubgroup(QUEUE_LABEL, QueueName)) {
- QueueCounters.SqsCounters->ResetCounters();
- parent.SqsCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.SqsCounters);
+ if (!parent.SqsCounters->FindSubgroup(QUEUE_LABEL, QueueName)) {
+ QueueCounters.SqsCounters->ResetCounters();
+ parent.SqsCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.SqsCounters);
+ }
+ if (parent.YmqCounters && !parent.YmqCounters->FindSubgroup(QUEUE_LABEL, QueueName)) {
+ QueueCounters.YmqCounters->ResetCounters();
+ parent.YmqCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.YmqCounters);
}
- if (parent.YmqCounters && !parent.YmqCounters->FindSubgroup(QUEUE_LABEL, QueueName)) {
- QueueCounters.YmqCounters->ResetCounters();
- parent.YmqCounters->RegisterSubgroup(QUEUE_LABEL, QueueName, QueueCounters.YmqCounters);
- }
};
- if (FolderCounters.Defined()) {
+ if (FolderCounters.Defined()) {
insert(FolderCounters);
} else {
insert(UserCounters);
@@ -713,15 +713,15 @@ void TQueueCounters::SetAggregatedParent(const TIntrusivePtr<TQueueCounters>& pa
QueueCountersDescriptor.SetAggregatedParent(this, parent.Get());
DetailedCounters.SetAggregatedParent(parent ? &parent->DetailedCounters : nullptr);
for (size_t i = 0; i < EAction::ActionsArraySize; ++i) {
- SqsActionCounters[i].SetAggregatedParent(parent ? &parent->SqsActionCounters[i] : nullptr);
+ SqsActionCounters[i].SetAggregatedParent(parent ? &parent->SqsActionCounters[i] : nullptr);
}
}
void TQueueCounters::RemoveCounters() {
- auto couple = FolderCounters.Defined() ? FolderCounters : UserCounters;
- couple.SqsCounters->RemoveSubgroup(QUEUE_LABEL, QueueName);
- if (couple.YmqCounters)
- couple.YmqCounters->RemoveSubgroup(QUEUE_LABEL, QueueName);
+ auto couple = FolderCounters.Defined() ? FolderCounters : UserCounters;
+ couple.SqsCounters->RemoveSubgroup(QUEUE_LABEL, QueueName);
+ if (couple.YmqCounters)
+ couple.YmqCounters->RemoveSubgroup(QUEUE_LABEL, QueueName);
}
TIntrusivePtr<TQueueCounters> TQueueCounters::GetCountersForLeaderNode() {
@@ -741,22 +741,22 @@ TIntrusivePtr<TQueueCounters> TQueueCounters::GetCountersForNotLeaderNode() {
void TUserCounters::InitCounters(const TString& userName, const std::shared_ptr<TAlignedPagePoolCounters>& allocPoolCounters) {
UserCounters = GetUserCounters(SqsCoreCounters, userName);
- INIT_COUNTER(UserCounters.SqsCounters, RequestTimeouts, ELifetime::Persistent, EValueType::Derivative, Lazy(*Cfg));
+ INIT_COUNTER(UserCounters.SqsCounters, RequestTimeouts, ELifetime::Persistent, EValueType::Derivative, Lazy(*Cfg));
if (Cfg->GetForceAccessControl() && Cfg->AccountsWithoutMandatoryAuthSize() && (userName == TOTAL_COUNTER_LABEL || IsIn(Cfg->GetAccountsWithoutMandatoryAuth(), userName))) {
- INIT_COUNTER(UserCounters.SqsCounters, UnauthenticatedAccess, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart);
+ INIT_COUNTER(UserCounters.SqsCounters, UnauthenticatedAccess, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnStart);
}
for (EAction action = static_cast<EAction>(EAction::Unknown + 1); action < EAction::ActionsArraySize; action = static_cast<EAction>(action + 1)) {
if (IsActionForUser(action)) {
- SqsActionCounters[action].Init(*Cfg, UserCounters.SqsCounters, action);
+ SqsActionCounters[action].Init(*Cfg, UserCounters.SqsCounters, action);
+ }
+ if (IsActionForUserYMQ(action) && UserCounters.YmqCounters && !IsAggregatedCounters) {
+ YmqActionCounters[action].Init(*Cfg, UserCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX);
}
- if (IsActionForUserYMQ(action) && UserCounters.YmqCounters && !IsAggregatedCounters) {
- YmqActionCounters[action].Init(*Cfg, UserCounters.YmqCounters, action, METHOD_LABLE, ACTION_CNTR_PREFIX);
- }
}
- // ToDo. Errors codes here. Will probably need this in Ymq counters further
+ // ToDo. Errors codes here. Will probably need this in Ymq counters further
DetailedCounters.Init(UserCounters, allocPoolCounters, *Cfg);
AggregatedQueueCounters = CreateQueueCountersImpl(TOTAL_COUNTER_LABEL, Cfg->GetYandexCloudMode() ? TOTAL_COUNTER_LABEL : TString(), true, true)->GetCountersForLeaderNode();
@@ -766,7 +766,7 @@ void TUserCounters::InitCounters(const TString& userName, const std::shared_ptr<
UserCountersDescriptor.SetAggregatedParent(this, AggregatedParent.Get());
DetailedCounters.SetAggregatedParent(&AggregatedParent->DetailedCounters);
for (size_t i = 0; i < EAction::ActionsArraySize; ++i) {
- SqsActionCounters[i].SetAggregatedParent(&AggregatedParent->SqsActionCounters[i]);
+ SqsActionCounters[i].SetAggregatedParent(&AggregatedParent->SqsActionCounters[i]);
}
}
}
@@ -776,13 +776,13 @@ void TUserCounters::TDetailedCounters::Init(const TIntrusivePtrCntrCouple& userC
TransactionCounters = new TTransactionCounters();
TransactionCounters->Init(GetAggregatedCountersFromUserCounters(userCounters, cfg), allocPoolCounters, false);
- APIStatuses.Init(userCounters.SqsCounters);
+ APIStatuses.Init(userCounters.SqsCounters);
- INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetConfiguration_Duration, ELifetime::Persistent, DurationBucketsMs, ELaziness::OnDemand);
- INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetQuota_Duration, ELifetime::Persistent, GetQuotaDurationBucketsMs, ELaziness::OnDemand);
+ INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetConfiguration_Duration, ELifetime::Persistent, DurationBucketsMs, ELaziness::OnDemand);
+ INIT_HISTOGRAM_COUNTER(userCounters.SqsCounters, GetQuota_Duration, ELifetime::Persistent, GetQuotaDurationBucketsMs, ELaziness::OnDemand);
- INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Success, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand);
- INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Errors, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand);
+ INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Success, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand);
+ INIT_COUNTER(userCounters.SqsCounters, CreateAccountOnTheFly_Errors, ELifetime::Persistent, EValueType::Derivative, ELaziness::OnDemand);
}
void TUserCounters::TDetailedCounters::SetAggregatedParent(TUserCounters::TDetailedCounters* parent) {
@@ -791,7 +791,7 @@ void TUserCounters::TDetailedCounters::SetAggregatedParent(TUserCounters::TDetai
}
TIntrusivePtr<TQueueCounters> TUserCounters::CreateQueueCounters(const TString& queueName, const TString& folderId, bool insertCounters) {
- auto counters = CreateQueueCountersImpl(queueName, folderId, insertCounters, IsAggregatedCounters);
+ auto counters = CreateQueueCountersImpl(queueName, folderId, insertCounters, IsAggregatedCounters);
counters->SetAggregatedParent(AggregatedQueueCounters);
return counters;
}
@@ -801,25 +801,25 @@ TIntrusivePtr<TQueueCounters> TUserCounters::CreateQueueCountersImpl(const TStri
}
void TUserCounters::RemoveCounters() {
- SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName);
- if (SqsCoreCounters.YmqCounters)
- SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName);
+ SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName);
+ if (SqsCoreCounters.YmqCounters)
+ SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName);
}
void TUserCounters::DisableCounters(bool disable) {
if (disable) {
- SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName);
- if (SqsCoreCounters.YmqCounters)
- SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName);
+ SqsCoreCounters.SqsCounters->RemoveSubgroup(USER_LABEL, UserName);
+ if (SqsCoreCounters.YmqCounters)
+ SqsCoreCounters.YmqCounters->RemoveSubgroup(CLOUD_LABEL, UserName);
} else {
- if (!SqsCoreCounters.SqsCounters->FindSubgroup(USER_LABEL, UserName)) {
- UserCounters.SqsCounters->ResetCounters();
- SqsCoreCounters.SqsCounters->RegisterSubgroup(USER_LABEL, UserName, UserCounters.SqsCounters);
+ if (!SqsCoreCounters.SqsCounters->FindSubgroup(USER_LABEL, UserName)) {
+ UserCounters.SqsCounters->ResetCounters();
+ SqsCoreCounters.SqsCounters->RegisterSubgroup(USER_LABEL, UserName, UserCounters.SqsCounters);
+ }
+ if (SqsCoreCounters.YmqCounters && !SqsCoreCounters.YmqCounters->FindSubgroup(CLOUD_LABEL, UserName)) {
+ UserCounters.YmqCounters->ResetCounters();
+ SqsCoreCounters.YmqCounters->RegisterSubgroup(CLOUD_LABEL, UserName, UserCounters.YmqCounters);
}
- if (SqsCoreCounters.YmqCounters && !SqsCoreCounters.YmqCounters->FindSubgroup(CLOUD_LABEL, UserName)) {
- UserCounters.YmqCounters->ResetCounters();
- SqsCoreCounters.YmqCounters->RegisterSubgroup(CLOUD_LABEL, UserName, UserCounters.YmqCounters);
- }
}
}
diff --git a/ydb/core/ymq/base/counters.h b/ydb/core/ymq/base/counters.h
index ecd9777af2..3c2667b241 100644
--- a/ydb/core/ymq/base/counters.h
+++ b/ydb/core/ymq/base/counters.h
@@ -29,12 +29,12 @@ struct TCloudAuthCounters;
++*countersPack->counter; \
}
-#define INC_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter) \
- if (countersPack) { \
- ++*countersPack->sqsCounter; \
- ++*countersPack->ymqCounter; \
- }
-
+#define INC_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter) \
+ if (countersPack) { \
+ ++*countersPack->sqsCounter; \
+ ++*countersPack->ymqCounter; \
+ }
+
#define DEC_COUNTER(countersPack, counter) \
if (countersPack) { \
--*countersPack->counter; \
@@ -45,31 +45,31 @@ struct TCloudAuthCounters;
*countersPack->counter += (count); \
}
-#define ADD_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \
- if (countersPack) { \
- *countersPack->sqsCounter += (count); \
- *countersPack->ymqCounter += (count); \
- }
-
-#define SET_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \
- if (countersPack) { \
- *countersPack->sqsCounter = (count); \
- *countersPack->ymqCounter = (count); \
- }
-
+#define ADD_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \
+ if (countersPack) { \
+ *countersPack->sqsCounter += (count); \
+ *countersPack->ymqCounter += (count); \
+ }
+
+#define SET_COUNTER_COUPLE(countersPack, sqsCounter, ymqCounter, count) \
+ if (countersPack) { \
+ *countersPack->sqsCounter = (count); \
+ *countersPack->ymqCounter = (count); \
+ }
+
#define COLLECT_HISTOGRAM_COUNTER(countersPack, counter, count) \
if (countersPack) { \
countersPack->counter->Collect(count); \
}
-#define COLLECT_HISTOGRAM_COUNTER_COUPLE(countersCouple, counter, count) \
- if (countersCouple.SqsCounters) { \
- countersCouple.SqsCounters->counter->Collect(count); \
- } \
- if (countersCouple.YmqCounters) { \
- countersCouple.YmqCounters->counter->Collect(count); \
- }
-
+#define COLLECT_HISTOGRAM_COUNTER_COUPLE(countersCouple, counter, count) \
+ if (countersCouple.SqsCounters) { \
+ countersCouple.SqsCounters->counter->Collect(count); \
+ } \
+ if (countersCouple.YmqCounters) { \
+ countersCouple.YmqCounters->counter->Collect(count); \
+ }
+
// Enums for code readability and for static types guarantee that all parameters passed correctly.
enum class ELaziness {
OnStart,
@@ -87,36 +87,36 @@ enum class EValueType {
};
extern const TString TOTAL_COUNTER_LABEL;
-//constexpr static std::array<int, 10> RESPONSE_CODES = {200, 400, 403, 404, 500, 503, 504};
-
-template<typename TCounterPtrType>
-struct TCountersCouple {
- TCounterPtrType SqsCounters = nullptr;
- TCounterPtrType YmqCounters = nullptr;
- bool Defined() {
- return SqsCounters != nullptr;
- }
-};
-
-using TIntrusivePtrCntrCouple = TCountersCouple<TIntrusivePtr<NMonitoring::TDynamicCounters>>;
-
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(
- const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup);
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(
- const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot);
-
-TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId);
-void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId);
-std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters(
- const TIntrusivePtrCntrCouple& sqsRootCounters, const TQueuePath& queuePath);
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters(
- const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg);
-TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters(
- const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg);
-
+//constexpr static std::array<int, 10> RESPONSE_CODES = {200, 400, 403, 404, 500, 503, 504};
+
+template<typename TCounterPtrType>
+struct TCountersCouple {
+ TCounterPtrType SqsCounters = nullptr;
+ TCounterPtrType YmqCounters = nullptr;
+ bool Defined() {
+ return SqsCounters != nullptr;
+ }
+};
+
+using TIntrusivePtrCntrCouple = TCountersCouple<TIntrusivePtr<NMonitoring::TDynamicCounters>>;
+
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetSqsServiceCounters(
+ const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot, const TString& subgroup);
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetYmqPublicCounters(
+ const TIntrusivePtr<NMonitoring::TDynamicCounters>& countersRoot);
+
+TIntrusivePtrCntrCouple GetFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId);
+void RemoveFolderCounters(const TIntrusivePtrCntrCouple& userCounters, const TString& folderId);
+std::pair<TIntrusivePtrCntrCouple, TIntrusivePtrCntrCouple> GetUserAndQueueCounters(
+ const TIntrusivePtrCntrCouple& sqsRootCounters, const TQueuePath& queuePath);
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromSqsCoreCounters(
+ const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg);
+TIntrusivePtr<NMonitoring::TDynamicCounters> GetAggregatedCountersFromUserCounters(
+ const TIntrusivePtrCntrCouple& sqsCoreCounters, const NKikimrConfig::TSqsConfig& cfg);
+
extern const TString DEFAULT_COUNTER_NAME;
-extern const TString DEFAULT_YMQ_COUNTER_NAME;
-//extern const TString ACTION_CNTR_PREFIX;
+extern const TString DEFAULT_YMQ_COUNTER_NAME;
+//extern const TString ACTION_CNTR_PREFIX;
namespace NDetails {
@@ -342,7 +342,7 @@ struct TLazyCachedHistogram : public NDetails::TLazyCachedCounterBase<NMonitorin
}
void Init(const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters, ELifetime lifetime, const NMonitoring::TBucketBounds& buckets, const TString& name, const TString& value, ELaziness laziness) {
- Buckets = &buckets;
+ Buckets = &buckets;
TLazyCachedCounterBase::Init(rootCounters, lifetime, name, value, ELaziness::OnDemand);
if (laziness == ELaziness::OnStart) {
EnsureCreated();
@@ -368,7 +368,7 @@ private:
// Counters for actions (like SendMessage, CreateQueue or GetQueueUrl).
struct TActionCounters {
-public:
+public:
TLazyCachedCounter Success;
TLazyCachedCounter Errors; // User metric for cloud console (SendMessage/ReceiveMessage/DeleteMessage).
TLazyCachedCounter Infly;
@@ -379,27 +379,27 @@ public:
TLazyCachedHistogram WorkingDuration; // Special duration except wait time for ReceiveMessage action (== 18 counters). // User metric for cloud console (ReceiveMessage).
public:
- void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters,
- EAction action, ELifetime lifetime = ELifetime::Persistent);
+ void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters,
+ EAction action, ELifetime lifetime = ELifetime::Persistent);
- virtual void SetAggregatedParent(TActionCounters* parent);
+ virtual void SetAggregatedParent(TActionCounters* parent);
virtual ~TActionCounters() {}
};
-struct TYmqActionCounters : public TActionCounters {
- void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters,
- EAction action, const TString& labelName, const TString& namePrefix,
- ELifetime lifetime = ELifetime::Persistent);
-public:
+struct TYmqActionCounters : public TActionCounters {
+ void Init(const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& rootCounters,
+ EAction action, const TString& labelName, const TString& namePrefix,
+ ELifetime lifetime = ELifetime::Persistent);
+public:
virtual ~TYmqActionCounters() {}
- void SetAggregatedParent(TActionCounters*) override {
- return;
- }
-
-private:
- NMonitoring::TDynamicCounterPtr SubGroup;
-};
-
+ void SetAggregatedParent(TActionCounters*) override {
+ return;
+ }
+
+private:
+ NMonitoring::TDynamicCounterPtr SubGroup;
+};
+
// Counters for typed queries (WRITE_MESSAGE_ID, PURGE_QUEUE_ID and etc).
struct TQueryTypeCounters {
TLazyCachedCounter TransactionsCount;
@@ -453,7 +453,7 @@ private:
struct TUserCounters : public TAtomicRefCount<TUserCounters> {
// Action types are declared in ydb/core/ymq/base/action.h.
TActionCounters SqsActionCounters[EAction::ActionsArraySize]; // 11 actions. See IsActionForUser() function in ydb/core/ymq/base/action.cpp.
- TYmqActionCounters YmqActionCounters[EAction::ActionsArraySize]; // see IsActionForUserYMQ() function.
+ TYmqActionCounters YmqActionCounters[EAction::ActionsArraySize]; // see IsActionForUserYMQ() function.
TLazyCachedCounter RequestTimeouts; // Requests that weren't processed in 10 minutes. They are almost sure hanged.
@@ -470,7 +470,7 @@ struct TUserCounters : public TAtomicRefCount<TUserCounters> {
TLazyCachedCounter CreateAccountOnTheFly_Success; // Account created on the fly (Yandex Cloud mode).
TLazyCachedCounter CreateAccountOnTheFly_Errors; // Account that were failed to create on the fly (Yandex Cloud mode).
- void Init(const TIntrusivePtrCntrCouple& userCounters,
+ void Init(const TIntrusivePtrCntrCouple& userCounters,
const std::shared_ptr<TAlignedPagePoolCounters>& allocPoolCounters,
const NKikimrConfig::TSqsConfig& cfg);
void SetAggregatedParent(TDetailedCounters* parent);
@@ -488,21 +488,21 @@ struct TUserCounters : public TAtomicRefCount<TUserCounters> {
// Raw counters interface
// Don't use counters by name!
- TIntrusivePtrCntrCouple SqsCoreCounters; // Sqs core subsystem
- TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem
+ TIntrusivePtrCntrCouple SqsCoreCounters; // Sqs core subsystem
+ TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem
- TUserCounters(
- const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsCoreCounters,
- const TIntrusivePtr<NMonitoring::TDynamicCounters>& ymqRootCounters,
+ TUserCounters(
+ const NKikimrConfig::TSqsConfig& cfg, const TIntrusivePtr<NMonitoring::TDynamicCounters>& sqsCoreCounters,
+ const TIntrusivePtr<NMonitoring::TDynamicCounters>& ymqRootCounters,
const std::shared_ptr<TAlignedPagePoolCounters>& allocPoolCounters, const TString& userName,
- const TIntrusivePtr<TUserCounters>& aggregatedParent,
- bool isAggregatedCounters = false
- )
- : SqsCoreCounters{sqsCoreCounters, ymqRootCounters}
+ const TIntrusivePtr<TUserCounters>& aggregatedParent,
+ bool isAggregatedCounters = false
+ )
+ : SqsCoreCounters{sqsCoreCounters, ymqRootCounters}
, Cfg(&cfg)
, UserName(userName)
, AggregatedParent(aggregatedParent)
- , IsAggregatedCounters(isAggregatedCounters)
+ , IsAggregatedCounters(isAggregatedCounters)
{
InitCounters(userName, allocPoolCounters);
}
@@ -551,7 +551,7 @@ private:
TDetailedCounters DetailedCounters;
TIntrusivePtr<TUserCounters> AggregatedParent;
TIntrusivePtr<TQueueCounters> AggregatedQueueCounters;
- bool IsAggregatedCounters;
+ bool IsAggregatedCounters;
};
// Queue counters in SQS core subsystem.
@@ -561,46 +561,46 @@ struct TQueueCounters : public TAtomicRefCount<TQueueCounters> {
TYmqActionCounters YmqActionCounters[EAction::ActionsArraySize]; // See IsActionForQueueYMQ() function in ydb/core/ymq/sqs/base/action.cpp.
TLazyCachedCounter RequestTimeouts; // Requests that weren't processed in 10 minutes. They are almost sure hanged.
- TLazyCachedCounter request_timeouts_count_per_second; // Requests that weren't processed in 10 minutes. They are almost sure hanged.
+ TLazyCachedCounter request_timeouts_count_per_second; // Requests that weren't processed in 10 minutes. They are almost sure hanged.
TLazyCachedCounter RequestsThrottled; // Request that ended with ThrottlingException
TLazyCachedCounter QueueMasterStartProblems; // TODO: remove after migration
TLazyCachedCounter QueueLeaderStartProblems; // Critical problems during leader start.
TLazyCachedCounter MessagesPurged;
- TLazyCachedCounter purged_count_per_second;
+ TLazyCachedCounter purged_count_per_second;
TLazyCachedHistogram MessageReceiveAttempts; // User attempts for receive messages. Histogram with buckets for receive attempts (== 4 counters). // User metric for cloud console.
- TLazyCachedHistogram receive_attempts_count_rate;
+ TLazyCachedHistogram receive_attempts_count_rate;
TLazyCachedHistogram ClientMessageProcessing_Duration; // Time between receive and delete for deleted message. Histogram with buckets for client processing (== 21 counters). // User metric for cloud console.
- TLazyCachedHistogram client_processing_duration_milliseconds;
+ TLazyCachedHistogram client_processing_duration_milliseconds;
TLazyCachedHistogram MessageReside_Duration; // Time between send and receive for received messages. Histogram with buckets for client processing (== 21 counters). // User metric for cloud console.
- TLazyCachedHistogram reside_duration_milliseconds;
+ TLazyCachedHistogram reside_duration_milliseconds;
TLazyCachedCounter DeleteMessage_Count; // Messages count that were deleted. // User metric for cloud console.
- TLazyCachedCounter deleted_count_per_second;
+ TLazyCachedCounter deleted_count_per_second;
TLazyCachedCounter ReceiveMessage_EmptyCount; // Receive message requests count that returned empty results.
- TLazyCachedCounter empty_receive_attempts_count_per_second;
+ TLazyCachedCounter empty_receive_attempts_count_per_second;
TLazyCachedCounter ReceiveMessage_Count; // Messages count that were received. // User metric for cloud console.
- TLazyCachedCounter received_count_per_second;
+ TLazyCachedCounter received_count_per_second;
TLazyCachedCounter ReceiveMessage_BytesRead; // Bytes of message bodies that were received. // User metric for cloud console.
- TLazyCachedCounter received_bytes_per_second;
+ TLazyCachedCounter received_bytes_per_second;
TLazyCachedCounter MessagesMovedToDLQ; // Count of messages that were moved to DLQ.
TLazyCachedCounter SendMessage_DeduplicationCount; // Count of messages that were deduplicated (for fifo only).
- TLazyCachedCounter deduplicated_count_per_second;
+ TLazyCachedCounter deduplicated_count_per_second;
TLazyCachedCounter SendMessage_Count; // Count of messages that were sent. // User metric for cloud console.
- TLazyCachedCounter sent_count_per_second;
+ TLazyCachedCounter sent_count_per_second;
TLazyCachedCounter SendMessage_BytesWritten; // Bytes of message bodies that were sent.
- TLazyCachedCounter sent_bytes_per_second;
+ TLazyCachedCounter sent_bytes_per_second;
TLazyCachedCounter MessagesCount; // Messages count in queue. // User metric for cloud console.
- TLazyCachedCounter stored_count;
+ TLazyCachedCounter stored_count;
TLazyCachedCounter InflyMessagesCount; // Messages count in queue that are inflight. // User metric for cloud console.
- TLazyCachedCounter inflight_count;
+ TLazyCachedCounter inflight_count;
TLazyCachedCounter OldestMessageAgeSeconds; // Age of the oldest message in queue. // User metric for cloud console.
- TLazyCachedCounter oldest_age_milliseconds;
+ TLazyCachedCounter oldest_age_milliseconds;
struct TDetailedCounters {
TIntrusivePtr<TTransactionCounters> TransactionCounters;
@@ -626,16 +626,16 @@ struct TQueueCounters : public TAtomicRefCount<TQueueCounters> {
}
}
- // Raw counters interface.RootCounters
+ // Raw counters interface.RootCounters
// Don't use counters by name!
- TIntrusivePtrCntrCouple RootCounters; // Sqs core subsystem.
- TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem
- TIntrusivePtrCntrCouple FolderCounters; // Folder subtree in user tree (only for Yandex Cloud).
- TIntrusivePtrCntrCouple QueueCounters; // Queue subtree in user (or folder) tree.
+ TIntrusivePtrCntrCouple RootCounters; // Sqs core subsystem.
+ TIntrusivePtrCntrCouple UserCounters; // User tree in core subsystem
+ TIntrusivePtrCntrCouple FolderCounters; // Folder subtree in user tree (only for Yandex Cloud).
+ TIntrusivePtrCntrCouple QueueCounters; // Queue subtree in user (or folder) tree.
// Creates counters for not leader node.
TQueueCounters(const NKikimrConfig::TSqsConfig& cfg,
- const TIntrusivePtrCntrCouple& sqsCoreCounters,
+ const TIntrusivePtrCntrCouple& sqsCoreCounters,
const TUserCounters* userCounters,
const TString& queueName,
const TString& folderId,
diff --git a/ydb/core/ymq/base/ut/counters_ut.cpp b/ydb/core/ymq/base/ut/counters_ut.cpp
index f2189504bc..5ff3d26e36 100644
--- a/ydb/core/ymq/base/ut/counters_ut.cpp
+++ b/ydb/core/ymq/base/ut/counters_ut.cpp
@@ -195,99 +195,99 @@ Y_UNIT_TEST_SUITE(LazyCounterTest) {
}
Y_UNIT_TEST_SUITE(UserCountersTest) {
-#define ASSERT_STR_COUPLE_CONTAINS(string1, string2, what) \
- UNIT_ASSERT_STRING_CONTAINS(string1, what); \
- UNIT_ASSERT_STRING_CONTAINS(string2, what);
-
-#define ASSERT_FIRST_OF_COUPLE_CONTAINS(string1, string2, what) \
- UNIT_ASSERT_STRING_CONTAINS(string1, what); \
- UNIT_ASSERT(!string2.Contains(what));
-
-#define ASSERT_STR_COUPLE_DONT_CONTAIN(string1, string2, what) \
- UNIT_ASSERT(string1.find(what) == TString::npos); \
- UNIT_ASSERT(string2.find(what) == TString::npos);
-
-#define ASSERT_USER_PRESENT(user) \
- ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \
- ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(ymqCounters), CountersString(core), TString("cloud=") + user);
-
-#define ASSERT_USER_ABSENT(user) \
- ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \
- ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("cloud=") + user);
-
+#define ASSERT_STR_COUPLE_CONTAINS(string1, string2, what) \
+ UNIT_ASSERT_STRING_CONTAINS(string1, what); \
+ UNIT_ASSERT_STRING_CONTAINS(string2, what);
+
+#define ASSERT_FIRST_OF_COUPLE_CONTAINS(string1, string2, what) \
+ UNIT_ASSERT_STRING_CONTAINS(string1, what); \
+ UNIT_ASSERT(!string2.Contains(what));
+
+#define ASSERT_STR_COUPLE_DONT_CONTAIN(string1, string2, what) \
+ UNIT_ASSERT(string1.find(what) == TString::npos); \
+ UNIT_ASSERT(string2.find(what) == TString::npos);
+
+#define ASSERT_USER_PRESENT(user) \
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(CountersString(ymqCounters), CountersString(core), TString("cloud=") + user);
+
+#define ASSERT_USER_ABSENT(user) \
+ ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("user=") + user); \
+ ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), TString("cloud=") + user);
+
Y_UNIT_TEST(DisableCountersTest) {
NKikimrConfig::TSqsConfig cfg;
cfg.SetCreateLazyCounters(false);
TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<TUserCounters> user1 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user1", nullptr);
- TIntrusivePtr<TUserCounters> user2 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user2", nullptr);
- ASSERT_USER_PRESENT("user1");
- ASSERT_USER_PRESENT("user2");
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
+ TIntrusivePtr<TUserCounters> user1 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user1", nullptr);
+ TIntrusivePtr<TUserCounters> user2 = new TUserCounters(cfg, core, ymqCounters, nullptr, "user2", nullptr);
+ ASSERT_USER_PRESENT("user1");
+ ASSERT_USER_PRESENT("user2");
user1->DisableCounters(false);
- ASSERT_USER_PRESENT("user1");
+ ASSERT_USER_PRESENT("user1");
user1->DisableCounters(true);
- TString sqsCntrText = CountersString(core);
- TString ymqCntrText = CountersString(ymqCounters);
- ASSERT_USER_ABSENT("user1");
- ASSERT_USER_PRESENT("user2");
+ TString sqsCntrText = CountersString(core);
+ TString ymqCntrText = CountersString(ymqCounters);
+ ASSERT_USER_ABSENT("user1");
+ ASSERT_USER_PRESENT("user2");
// again
user1->DisableCounters(true);
- sqsCntrText = CountersString(core);
- ymqCntrText = CountersString(ymqCounters);
- ASSERT_USER_ABSENT("user1");
- ASSERT_USER_PRESENT("user2");
+ sqsCntrText = CountersString(core);
+ ymqCntrText = CountersString(ymqCounters);
+ ASSERT_USER_ABSENT("user1");
+ ASSERT_USER_PRESENT("user2");
*user1->RequestTimeouts = 1;
*user2->RequestTimeouts = 2;
// return back
user1->DisableCounters(false);
- sqsCntrText = CountersString(core);
- ymqCntrText = CountersString(ymqCounters);
- ASSERT_USER_PRESENT("user1");
- ASSERT_USER_PRESENT("user2");
- ASSERT_STR_COUPLE_DONT_CONTAIN(sqsCntrText,ymqCntrText, "RequestTimeouts: 1");
- ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2");
- ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2");
- ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 0");
+ sqsCntrText = CountersString(core);
+ ymqCntrText = CountersString(ymqCounters);
+ ASSERT_USER_PRESENT("user1");
+ ASSERT_USER_PRESENT("user2");
+ ASSERT_STR_COUPLE_DONT_CONTAIN(sqsCntrText,ymqCntrText, "RequestTimeouts: 1");
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2");
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2");
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 0");
// again
*user1->RequestTimeouts = 3;
user1->DisableCounters(false);
- sqsCntrText = CountersString(core);
- ymqCntrText = CountersString(ymqCounters);
- ASSERT_USER_PRESENT("user1");
- ASSERT_USER_PRESENT("user2");
- ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2");
- ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 3");
+ sqsCntrText = CountersString(core);
+ ymqCntrText = CountersString(ymqCounters);
+ ASSERT_USER_PRESENT("user1");
+ ASSERT_USER_PRESENT("user2");
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 2");
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 3");
}
Y_UNIT_TEST(RemoveUserCountersTest) {
NKikimrConfig::TSqsConfig cfg;
cfg.SetCreateLazyCounters(false);
TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
+
+ TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr);
+ ASSERT_USER_PRESENT("my_user");
- TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr);
- ASSERT_USER_PRESENT("my_user");
-
user->RemoveCounters();
- ASSERT_USER_ABSENT("my_user");
+ ASSERT_USER_ABSENT("my_user");
}
Y_UNIT_TEST(CountersAggregationTest) {
NKikimrConfig::TSqsConfig cfg;
cfg.SetCreateLazyCounters(false);
TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr);
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
+ TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr);
total->ShowDetailedCounters(TInstant::Max());
- TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total);
+ TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total);
UNIT_ASSERT_STRING_CONTAINS(CountersString(core), "user=my_user");
UNIT_ASSERT_STRING_CONTAINS(CountersString(core), "user=total");
@@ -300,7 +300,7 @@ Y_UNIT_TEST_SUITE(UserCountersTest) {
// Try different types of member counters: detailed/usual/arrays/api statuses.
++*user->RequestTimeouts;
- ++*user->SqsActionCounters[EAction::CreateQueue].Errors;
+ ++*user->SqsActionCounters[EAction::CreateQueue].Errors;
++*user->GetDetailedCounters()->CreateAccountOnTheFly_Success;
user->GetDetailedCounters()->APIStatuses.AddOk(3);
user->GetDetailedCounters()->APIStatuses.AddError("AccessDeniedException", 2);
@@ -308,8 +308,8 @@ Y_UNIT_TEST_SUITE(UserCountersTest) {
++*user->GetTransactionCounters()->TransactionsCount;
++*user->GetTransactionCounters()->QueryTypeCounters[EQueryId::WRITE_MESSAGE_ID].TransactionsFailed;
- ++*user->YmqActionCounters[EAction::CreateQueue].Errors;
-
+ ++*user->YmqActionCounters[EAction::CreateQueue].Errors;
+
AssertCounterValues(core,
{
{ "user=total/RequestTimeouts", 1 },
@@ -331,14 +331,14 @@ Y_UNIT_TEST_SUITE(UserCountersTest) {
{ "user=total/queue=total/TransactionsFailedByType/query_type=WRITE_MESSAGE_ID", 1 },
{ "user=my_user/queue=total/TransactionsFailedByType/query_type=WRITE_MESSAGE_ID", 1 },
});
-
-// auto queue = user->CreateQueueCounters("my_queue", "folder", true);
-
- AssertCounterValues(ymqCounters,
- {
- { "cloud=my_user/method=create_queue/name=api.http.errors_count_per_second", 1 },
- }
- );
+
+// auto queue = user->CreateQueueCounters("my_queue", "folder", true);
+
+ AssertCounterValues(ymqCounters,
+ {
+ { "cloud=my_user/method=create_queue/name=api.http.errors_count_per_second", 1 },
+ }
+ );
}
}
@@ -347,44 +347,44 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) {
NKikimrConfig::TSqsConfig cfg;
cfg.SetCreateLazyCounters(false);
TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr);
- ASSERT_USER_PRESENT("my_user");
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
+ TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr);
+ ASSERT_USER_PRESENT("my_user");
- auto queue = user->CreateQueueCounters("my_queue", "folder", false);
- ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "my_queue");
+ auto queue = user->CreateQueueCounters("my_queue", "folder", false);
+ ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "my_queue");
queue->InsertCounters();
- ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue");
-
- ++*queue->RequestTimeouts;
- auto sqsCntrText = CountersString(core);
- auto ymqCntrText = CountersString(ymqCounters);
- ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 1");
-
- ++*queue->request_timeouts_count_per_second;
- sqsCntrText = CountersString(core);
- ymqCntrText = CountersString(ymqCounters);
- ASSERT_FIRST_OF_COUPLE_CONTAINS(ymqCntrText, sqsCntrText, "request_timeouts_count_per_second: 1");
-
+ ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue");
+
+ ++*queue->RequestTimeouts;
+ auto sqsCntrText = CountersString(core);
+ auto ymqCntrText = CountersString(ymqCounters);
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(sqsCntrText, ymqCntrText, "RequestTimeouts: 1");
+
+ ++*queue->request_timeouts_count_per_second;
+ sqsCntrText = CountersString(core);
+ ymqCntrText = CountersString(ymqCounters);
+ ASSERT_FIRST_OF_COUPLE_CONTAINS(ymqCntrText, sqsCntrText, "request_timeouts_count_per_second: 1");
+
// Second time:
queue->InsertCounters();
- ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue");
+ ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "my_queue");
}
void RemoveQueueCountersTest(bool leader, const TString& folderId) {
NKikimrConfig::TSqsConfig cfg;
cfg.SetCreateLazyCounters(false);
TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr);
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
+ TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", nullptr);
TIntrusivePtr<TQueueCounters> queue = user->CreateQueueCounters("my_queue", folderId, true);
if (leader) {
queue = queue->GetCountersForLeaderNode();
}
- ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "queue=my_queue");
+ ASSERT_STR_COUPLE_CONTAINS(CountersString(core), CountersString(ymqCounters), "queue=my_queue");
queue->RemoveCounters();
- ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "queue=my_queue");
+ ASSERT_STR_COUPLE_DONT_CONTAIN(CountersString(core), CountersString(ymqCounters), "queue=my_queue");
}
Y_UNIT_TEST(RemoveQueueCountersNonLeaderWithoutFolderTest) {
@@ -408,10 +408,10 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) {
cfg.SetYandexCloudMode(cloudMode);
cfg.SetCreateLazyCounters(false);
TIntrusivePtr<NMonitoring::TDynamicCounters> core = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
- TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr);
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters = new NMonitoring::TDynamicCounters();
+ TIntrusivePtr<TUserCounters> total = new TUserCounters(cfg, core, ymqCounters, nullptr, TOTAL_COUNTER_LABEL, nullptr);
total->ShowDetailedCounters(TInstant::Max());
- TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total);
+ TIntrusivePtr<TUserCounters> user = new TUserCounters(cfg, core, ymqCounters, nullptr, "my_user", total);
TIntrusivePtr<TQueueCounters> queue = user->CreateQueueCounters("my_queue", cloudMode ? "my_folder" : "", true)->GetCountersForLeaderNode();
UNIT_ASSERT_STRING_CONTAINS(CountersString(core), "queue=my_queue");
@@ -419,9 +419,9 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) {
// Try different types of member counters: detailed/usual/arrays/api statuses.
++*queue->MessagesPurged;
- ++*queue->purged_count_per_second;
- ++*queue->SqsActionCounters[EAction::SendMessageBatch].Errors;
- ++*queue->YmqActionCounters[EAction::SendMessageBatch].Errors;
+ ++*queue->purged_count_per_second;
+ ++*queue->SqsActionCounters[EAction::SendMessageBatch].Errors;
+ ++*queue->YmqActionCounters[EAction::SendMessageBatch].Errors;
++*queue->GetDetailedCounters()->ReceiveMessage_KeysInvalidated;
++*queue->GetTransactionCounters()->TransactionRetryTimeouts;
++*queue->GetTransactionCounters()->QueryTypeCounters[EQueryId::DELETE_MESSAGE_ID].TransactionsCount;
@@ -459,13 +459,13 @@ Y_UNIT_TEST_SUITE(QueueCountersTest) {
{ x("user=my_user/queue=total/TransactionsByType/query_type=DELETE_MESSAGE_ID"), 1 },
{ x("user=my_user/queue=my_queue/TransactionsByType/query_type=DELETE_MESSAGE_ID"), 1 },
});
-
- AssertCounterValues(ymqCounters,
- {
- { x("cloud=my_user/queue=my_queue/name=queue.messages.purged_count_per_second"), 1 },
-
- { x("cloud=my_user/queue=my_queue/method=send_message_batch/name=api.http.errors_count_per_second"), 1 },
- });
+
+ AssertCounterValues(ymqCounters,
+ {
+ { x("cloud=my_user/queue=my_queue/name=queue.messages.purged_count_per_second"), 1 },
+
+ { x("cloud=my_user/queue=my_queue/method=send_message_batch/name=api.http.errors_count_per_second"), 1 },
+ });
}
Y_UNIT_TEST(CountersAggregationTest) {
diff --git a/ydb/core/ymq/http/http.cpp b/ydb/core/ymq/http/http.cpp
index 1418127cd5..5d0963bfb2 100644
--- a/ydb/core/ymq/http/http.cpp
+++ b/ydb/core/ymq/http/http.cpp
@@ -233,12 +233,12 @@ bool THttpRequest::DoReply(const TReplyParams& p) {
const TDuration parseTime = TInstant::Now() - StartTime_;
RLOG_SQS_BASE_DEBUG(*Parent_->ActorSystem_, "Parse time: [" << parseTime.MilliSeconds() << "ms]");
- RLOG_SQS_BASE_INFO(
- *Parent_->ActorSystem_,
- "Start request. User [" << UserName_ << "] Queue [" << QueueName_ << "], Cloud [" << AccountName_
- << "], Folder [" << FolderId_ << "] Action [" << ActionToString(Action_)
- << "] IP [" << SourceAddress_ << "]"
- );
+ RLOG_SQS_BASE_INFO(
+ *Parent_->ActorSystem_,
+ "Start request. User [" << UserName_ << "] Queue [" << QueueName_ << "], Cloud [" << AccountName_
+ << "], Folder [" << FolderId_ << "] Action [" << ActionToString(Action_)
+ << "] IP [" << SourceAddress_ << "]"
+ );
if (!Parent_->Config.GetYandexCloudMode() && UserName_.empty()) {
WriteResponse(p, MakeErrorXmlResponse(NErrors::MISSING_PARAMETER, Parent_->AggregatedUserCounters_.Get(), "No user name was provided."));
@@ -996,19 +996,19 @@ TAsyncHttpServer::~TAsyncHttpServer() {
DebugInfo->HttpServer = nullptr;
}
-void TAsyncHttpServer::Initialize(
- NActors::TActorSystem* as, TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters,
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters, ui32 poolId
-) {
+void TAsyncHttpServer::Initialize(
+ NActors::TActorSystem* as, TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters, ui32 poolId
+) {
ActorSystem_ = as;
- HttpCounters_ = new THttpCounters(Config, sqsCounters->GetSubgroup("subsystem", "http"));
+ HttpCounters_ = new THttpCounters(Config, sqsCounters->GetSubgroup("subsystem", "http"));
if (Config.GetYandexCloudMode()) {
- CloudAuthCounters_ = MakeHolder<TCloudAuthCounters>(Config, sqsCounters->GetSubgroup("subsystem", "cloud_auth"));
+ CloudAuthCounters_ = MakeHolder<TCloudAuthCounters>(Config, sqsCounters->GetSubgroup("subsystem", "cloud_auth"));
}
- AggregatedUserCounters_ = MakeIntrusive<TUserCounters>(
- Config, sqsCounters->GetSubgroup("subsystem", "core"), ymqCounters,
- nullptr, TOTAL_COUNTER_LABEL, nullptr, true
- );
+ AggregatedUserCounters_ = MakeIntrusive<TUserCounters>(
+ Config, sqsCounters->GetSubgroup("subsystem", "core"), ymqCounters,
+ nullptr, TOTAL_COUNTER_LABEL, nullptr, true
+ );
AggregatedUserCounters_->ShowDetailedCounters(TInstant::Max());
PoolId_ = poolId;
}
diff --git a/ydb/core/ymq/http/http.h b/ydb/core/ymq/http/http.h
index 28b140174c..01a2b0edc4 100644
--- a/ydb/core/ymq/http/http.h
+++ b/ydb/core/ymq/http/http.h
@@ -145,11 +145,11 @@ public:
TAsyncHttpServer(const NKikimrConfig::TSqsConfig& config);
~TAsyncHttpServer();
- void Initialize(
- NActors::TActorSystem* as,
- TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters,
- TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters,
- ui32 poolId);
+ void Initialize(
+ NActors::TActorSystem* as,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> sqsCounters,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> ymqCounters,
+ ui32 poolId);
void Start();
diff --git a/ydb/library/persqueue/topic_parser/topic_parser.cpp b/ydb/library/persqueue/topic_parser/topic_parser.cpp
index 46b20618eb..a9598b25b2 100644
--- a/ydb/library/persqueue/topic_parser/topic_parser.cpp
+++ b/ydb/library/persqueue/topic_parser/topic_parser.cpp
@@ -44,79 +44,79 @@ TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorCon
}
}
-TString MakeConsumerPath(const TString& consumer) {
- TStringBuilder res;
- res.reserve(consumer.size());
- for (ui32 i = 0; i < consumer.size(); ++i) {
- if (consumer[i] == '@') {
- res << "/";
- } else {
- res << consumer[i];
- }
- }
- if (res.find("/") == TString::npos) {
- return TStringBuilder() << "shared/" << res;
- }
- return res;
-}
-
-TString NormalizeFullPath(const TString& fullPath) {
- if (fullPath.StartsWith("/"))
- return fullPath.substr(1);
- else {
- return fullPath;
- }
-}
-
-TTopicsListController::TTopicsListController(
- const std::shared_ptr<TTopicNamesConverterFactory>& converterFactory,
- bool haveClusters, const TVector<TString>& clusters, const TString& localCluster
-)
- : ConverterFactory(converterFactory)
- , HaveClusters(haveClusters)
-{
- UpdateClusters(clusters, localCluster);
-}
-
-void TTopicsListController::UpdateClusters(const TVector<TString>& clusters, const TString& localCluster) {
- if (!HaveClusters)
- return;
-
- Clusters = clusters;
- LocalCluster = localCluster;
-}
-
-TTopicsToConverter TTopicsListController::GetReadTopicsList(
- const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const
-{
- TTopicsToConverter result;
- auto PutTopic = [&] (const TString& topic, const TString& dc) {
- auto converter = ConverterFactory->MakeTopicNameConverter(topic, dc, database);
- result[converter->GetPrimaryPath()] = converter;
- };
-
- for (const auto& t : clientTopics) {
- if (onlyLocal) {
- PutTopic(t, LocalCluster);
- } else if (HaveClusters){
- for(const auto& c : Clusters) {
- PutTopic(t, c);
- }
- } else {
- PutTopic(t, TString());
- }
- }
- return result;
-}
-TConverterPtr TTopicsListController::GetWriteTopicConverter(
- const TString& clientName, const TString& database
-) {
- return ConverterFactory->MakeTopicNameConverter(clientName, LocalCluster, database);
-}
-
-TConverterFactoryPtr TTopicsListController::GetConverterFactory() const {
- return ConverterFactory;
-};
-
+TString MakeConsumerPath(const TString& consumer) {
+ TStringBuilder res;
+ res.reserve(consumer.size());
+ for (ui32 i = 0; i < consumer.size(); ++i) {
+ if (consumer[i] == '@') {
+ res << "/";
+ } else {
+ res << consumer[i];
+ }
+ }
+ if (res.find("/") == TString::npos) {
+ return TStringBuilder() << "shared/" << res;
+ }
+ return res;
+}
+
+TString NormalizeFullPath(const TString& fullPath) {
+ if (fullPath.StartsWith("/"))
+ return fullPath.substr(1);
+ else {
+ return fullPath;
+ }
+}
+
+TTopicsListController::TTopicsListController(
+ const std::shared_ptr<TTopicNamesConverterFactory>& converterFactory,
+ bool haveClusters, const TVector<TString>& clusters, const TString& localCluster
+)
+ : ConverterFactory(converterFactory)
+ , HaveClusters(haveClusters)
+{
+ UpdateClusters(clusters, localCluster);
+}
+
+void TTopicsListController::UpdateClusters(const TVector<TString>& clusters, const TString& localCluster) {
+ if (!HaveClusters)
+ return;
+
+ Clusters = clusters;
+ LocalCluster = localCluster;
+}
+
+TTopicsToConverter TTopicsListController::GetReadTopicsList(
+ const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const
+{
+ TTopicsToConverter result;
+ auto PutTopic = [&] (const TString& topic, const TString& dc) {
+ auto converter = ConverterFactory->MakeTopicNameConverter(topic, dc, database);
+ result[converter->GetPrimaryPath()] = converter;
+ };
+
+ for (const auto& t : clientTopics) {
+ if (onlyLocal) {
+ PutTopic(t, LocalCluster);
+ } else if (HaveClusters){
+ for(const auto& c : Clusters) {
+ PutTopic(t, c);
+ }
+ } else {
+ PutTopic(t, TString());
+ }
+ }
+ return result;
+}
+TConverterPtr TTopicsListController::GetWriteTopicConverter(
+ const TString& clientName, const TString& database
+) {
+ return ConverterFactory->MakeTopicNameConverter(clientName, LocalCluster, database);
+}
+
+TConverterFactoryPtr TTopicsListController::GetConverterFactory() const {
+ return ConverterFactory;
+};
+
} // namespace NPersQueue
-
+
diff --git a/ydb/library/persqueue/topic_parser/topic_parser.h b/ydb/library/persqueue/topic_parser/topic_parser.h
index 18c55177f8..eb5a69cc37 100644
--- a/ydb/library/persqueue/topic_parser/topic_parser.h
+++ b/ydb/library/persqueue/topic_parser/topic_parser.h
@@ -3,352 +3,352 @@
#include <library/cpp/actors/core/actor.h>
#include <util/generic/string.h>
-#include <util/generic/hash.h>
-#include <util/string/builder.h>
+#include <util/generic/hash.h>
+#include <util/string/builder.h>
#include <ydb/core/base/path.h>
#include <ydb/library/persqueue/topic_parser_public/topic_parser.h>
-
+
namespace NPersQueue {
TString GetFullTopicPath(const NActors::TActorContext& ctx, TMaybe<TString> database, const TString& topicPath);
TString ConvertNewConsumerName(const TString& consumer, const NActors::TActorContext& ctx);
TString ConvertOldConsumerName(const TString& consumer, const NActors::TActorContext& ctx);
-TString MakeConsumerPath(const TString& consumer);
-
-
-#define CHECK_SET_VALID(cond, reason, statement) \
- if (!(cond)) { \
- Valid = false; \
- Reason = Reason + (reason) + ". "; \
- statement; \
- }
-
-
-#define CHECK_VALID_AND_RETURN(result) \
- if (!Valid) { \
- return TString(); \
- } else { \
- Y_VERIFY(!result.empty()); \
- return result; \
- }
-
-class TTopicNamesConverterFactory;
-
-class TTopicNameConverter {
- friend class TTopicNamesConverterFactory;
-
-public:
- TTopicNameConverter() = default;
-
-protected:
- TTopicNameConverter(bool noDcMode, TStringBuf& pqNormalizedRootPrefix)
- : NoDcMode(noDcMode)
- , PQRootPrefix(pqNormalizedRootPrefix)
- {}
-
- TTopicNameConverter& ApplyToTopic(
- const TString& topic, const TString& dc = TString(), const TString& database = TString()
- ) {
- OriginalTopic = topic;
-
- TStringBuf topicNormalized = topic;
- topicNormalized.SkipPrefix("/");
-
- TStringBuf dbNormalized = database;
- dbNormalized.SkipPrefix("/");
- dbNormalized.ChopSuffix("/");
-
- TStringBuf root;
- if (NoDcMode || !PQRootPrefix.StartsWith(dbNormalized)) {
- Database = database;
- root = dbNormalized;
- } else {
- root = PQRootPrefix;
- }
- topicNormalized.SkipPrefix(root);
- topicNormalized.SkipPrefix("/");
- Dc = dc;
- bool supposeLegacyMode = Database.empty() && !NoDcMode;
- auto hasDcInName = IsLegacyTopicNameWithDc(topicNormalized);
-
- CHECK_SET_VALID(!hasDcInName || dc.Empty(), "May not specify DC with legacy-style topic name containing dc", return *this);
- if (NoDcMode) {
- CHECK_SET_VALID(!Database.empty(), "Database is mandatory in FirstClass mode", return *this);
- } else {
- HasDc = hasDcInName || !dc.empty();
- }
-
- if (hasDcInName) {
- supposeLegacyMode = true;
- SplitFullLegacyName(topicNormalized);
- return *this;
- } else if (IsLegacyTopicName(topicNormalized)) { // Also must get here if 'SplitFullLegacyName' performed
- supposeLegacyMode = true;
- SplitShortLegacyName(topicNormalized);
- return *this;
- } else {
- SplitModernName(topicNormalized);
-
- BuildModernNames(TString(root));
- if (supposeLegacyMode) {
- ClientsideName = FullLegacyName;
- PrimaryFullPath = FullLegacyPath;
- } else {
- ClientsideName = ModernName;
- PrimaryFullPath = FullModernPath;
- }
- }
- return *this;
- }
-
-public:
- bool IsValid() const {
- return Valid;
- }
-
- const TString& GetReason() {
- return Reason;
- }
-
- TString GetClientsideName() {
- CHECK_VALID_AND_RETURN(ClientsideName);
- }
-
- TString GetPrimaryPath() {
- CHECK_VALID_AND_RETURN(PrimaryFullPath);
- }
-
- TString GetSecondaryPath() {
- Y_FAIL("UNIMPLEMENTED");
- CHECK_VALID_AND_RETURN(SecondaryFullPath);
- }
-
- TString GetOriginalPath() const {
- CHECK_VALID_AND_RETURN(OriginalTopic);
- }
-
- TString GetShortLegacyName() const {
- CHECK_VALID_AND_RETURN(LegacyName);
- }
-
- TString GetFullLegacyName() const {
- CHECK_VALID_AND_RETURN(FullLegacyName);
- }
-
- TString GetFullLegacyPath() const {
- CHECK_VALID_AND_RETURN(FullLegacyPath);
- }
- TString GetLegacyProducer() const {
- CHECK_VALID_AND_RETURN(Producer);
- }
-
- TString GetModernName() const {
- CHECK_VALID_AND_RETURN(ModernName);
- }
-
- TString GetModernPath() const {
- CHECK_VALID_AND_RETURN(FullModernPath);
- }
- TString GetCluster() const {
- return Dc;
- }
-
- TString GetAccount() const {
- if (!Valid)
- return {};
- if (!Database.empty()) {
- return Database;
- }
- TStringBuf buf(ModernName);
- TStringBuf fst, snd;
- auto res = buf.TrySplit("/", fst, snd);
- if (!res) {
- fst = buf;
- }
- return TString{fst};
- }
-
-private:
- bool IsLegacyTopicName(const TStringBuf& path) {
- if (path.find("--") != TStringBuf::npos)
- return true;
- if (Database.empty() && !NoDcMode && path.find("/") == TStringBuf::npos)
- return true;
- return false;
- }
-
- bool IsLegacyTopicNameWithDc(const TStringBuf& path) {
- return path.StartsWith("rt3.");
- }
-
- void SplitFullLegacyName(TStringBuf& name) {
- // rt3.sas--accountName[@dirName]--topicName
- FullLegacyName = name;
- TStringBuf fst, snd;
- name.SkipPrefix("rt3.");
- auto res = name.TrySplit("--", fst, snd);
- CHECK_SET_VALID(res, Sprintf("topic name %s is corrupted", OriginalTopic.c_str()), return);
- Dc = fst;
- LegacyName = snd;
- FullLegacyPath = NKikimr::JoinPath({TString(PQRootPrefix), FullLegacyName});
- SplitShortLegacyName(snd);
- }
-
- void SplitShortLegacyName(const TStringBuf& name) {
- if (LegacyName.empty())
- LegacyName = name;
- CHECK_SET_VALID(!NoDcMode, "legacy-style topic name as first-class citizen", return);
- TStringBuf buf, logtype;
- TStringBuilder topicPath;
- auto res = name.TryRSplit("--", buf, logtype);
- if (!res) {
- topicPath << name;
- Producer = name;
- } else {
- Producer = buf;
- while (true) {
- TStringBuf fst, snd;
- if (!buf.TrySplit("@", fst, snd))
- break;
- topicPath << fst << "/";
- buf = fst;
- }
- topicPath << buf << "/" << logtype;
- }
- ModernName = topicPath;
- BuildFullLegacyName();
- PrimaryFullPath = FullLegacyPath;
- if (ClientsideName.empty() && !NoDcMode && !FullLegacyName.empty()) {
- ClientsideName = FullLegacyName;
- }
- }
-
- void SplitModernName(TStringBuf& name) {
- if (ModernName.empty()) {
- ModernName = name;
- }
- TStringBuf fst, snd;
-
- auto res = name.TrySplit("/", fst, snd);
- if (!res) {
- LegacyName = name;
- Producer = name;
- } else {
- TStringBuilder legacyName;
- legacyName << fst;
- while (true) {
- name = snd;
- res = name.TrySplit("/", fst, snd);
- if (!res)
- break;
- legacyName << "@" << fst;
- }
- Producer = legacyName;
- legacyName << "--" << name;
- LegacyName = legacyName;
- }
- BuildFullLegacyName();
- }
-
- void BuildModernNames(const TString& prefix) {
- if (!NoDcMode && !Dc.empty()) {
- // ToDo[migration]: build full modern name, full modern path
- } else {
- FullModernName = ModernName;
- FullModernPath = NKikimr::JoinPath({prefix, FullModernName});
- }
- }
- void BuildFullLegacyName() {
- if (FullLegacyName.empty() || !Dc.empty()) {
- TStringBuilder builder;
- builder << "rt3." << Dc << "--" << LegacyName;
- FullLegacyName = builder;
- }
- if (FullLegacyPath.empty() && !FullLegacyName.empty()) {
- FullLegacyPath = NKikimr::JoinPath({PQRootPrefix, FullLegacyName});
- }
- }
-
-
- bool NoDcMode;
- TString PQRootPrefix;
-
- TString OriginalTopic;
- TString Database;
- TString Dc;
-
- TString LegacyName;
- TString FullLegacyName;
- TString FullLegacyPath;
-
- TString ModernName;
- TString FullModernName;
- TString FullModernPath;
-
- TString PrimaryFullPath;
- TString SecondaryFullPath;
- TString ClientsideName;
-
- TString Producer;
- TString Account;
- bool HasDc = false;
-
- bool Valid = true;
- TString Reason;
-};
-
-using TConverterPtr = std::shared_ptr<TTopicNameConverter>;
-
-class TTopicNamesConverterFactory {
-public:
- TTopicNamesConverterFactory(bool noDcMode, const TString& pqRootPrefix)
- : NoDcMode(noDcMode)
- , PQRootPrefix(pqRootPrefix)
- , NormalizedPrefix(PQRootPrefix)
- {
- NormalizedPrefix.SkipPrefix("/");
- NormalizedPrefix.ChopSuffix("/");
- }
-
- TConverterPtr MakeTopicNameConverter(
- const TString& topic, const TString& dc = TString(), const TString& database = TString()
- ) {
- auto* converter = new TTopicNameConverter(NoDcMode, NormalizedPrefix);
- converter->ApplyToTopic(topic, dc, database);
- return TConverterPtr(converter);
- }
-
-private:
- bool NoDcMode;
- TString PQRootPrefix;
- TStringBuf NormalizedPrefix;
-};
-
-using TConverterFactoryPtr = std::shared_ptr<NPersQueue::TTopicNamesConverterFactory>;
-using TTopicsToConverter = THashMap<TString, TConverterPtr>;
-
-class TTopicsListController {
-public:
- TTopicsListController(const TConverterFactoryPtr& converterFactory,
- bool haveClusters, const TVector<TString>& clusters = {}, const TString& localCluster = {});
-
- void UpdateClusters(const TVector<TString>& clusters, const TString& localCluster);
- TTopicsToConverter GetReadTopicsList(const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const;
- TConverterPtr GetWriteTopicConverter(const TString& clientName, const TString& database);
- TConverterFactoryPtr GetConverterFactory() const;
-
- bool GetHaveClusters() const { return HaveClusters;};
-
-private:
- TConverterFactoryPtr ConverterFactory;
- bool HaveClusters;
- TVector<TString> Clusters;
- TString LocalCluster;
-
-};
-
-TString NormalizeFullPath(const TString& fullPath);
-
-
+TString MakeConsumerPath(const TString& consumer);
+
+
+#define CHECK_SET_VALID(cond, reason, statement) \
+ if (!(cond)) { \
+ Valid = false; \
+ Reason = Reason + (reason) + ". "; \
+ statement; \
+ }
+
+
+#define CHECK_VALID_AND_RETURN(result) \
+ if (!Valid) { \
+ return TString(); \
+ } else { \
+ Y_VERIFY(!result.empty()); \
+ return result; \
+ }
+
+class TTopicNamesConverterFactory;
+
+class TTopicNameConverter {
+ friend class TTopicNamesConverterFactory;
+
+public:
+ TTopicNameConverter() = default;
+
+protected:
+ TTopicNameConverter(bool noDcMode, TStringBuf& pqNormalizedRootPrefix)
+ : NoDcMode(noDcMode)
+ , PQRootPrefix(pqNormalizedRootPrefix)
+ {}
+
+ TTopicNameConverter& ApplyToTopic(
+ const TString& topic, const TString& dc = TString(), const TString& database = TString()
+ ) {
+ OriginalTopic = topic;
+
+ TStringBuf topicNormalized = topic;
+ topicNormalized.SkipPrefix("/");
+
+ TStringBuf dbNormalized = database;
+ dbNormalized.SkipPrefix("/");
+ dbNormalized.ChopSuffix("/");
+
+ TStringBuf root;
+ if (NoDcMode || !PQRootPrefix.StartsWith(dbNormalized)) {
+ Database = database;
+ root = dbNormalized;
+ } else {
+ root = PQRootPrefix;
+ }
+ topicNormalized.SkipPrefix(root);
+ topicNormalized.SkipPrefix("/");
+ Dc = dc;
+ bool supposeLegacyMode = Database.empty() && !NoDcMode;
+ auto hasDcInName = IsLegacyTopicNameWithDc(topicNormalized);
+
+ CHECK_SET_VALID(!hasDcInName || dc.Empty(), "May not specify DC with legacy-style topic name containing dc", return *this);
+ if (NoDcMode) {
+ CHECK_SET_VALID(!Database.empty(), "Database is mandatory in FirstClass mode", return *this);
+ } else {
+ HasDc = hasDcInName || !dc.empty();
+ }
+
+ if (hasDcInName) {
+ supposeLegacyMode = true;
+ SplitFullLegacyName(topicNormalized);
+ return *this;
+ } else if (IsLegacyTopicName(topicNormalized)) { // Also must get here if 'SplitFullLegacyName' performed
+ supposeLegacyMode = true;
+ SplitShortLegacyName(topicNormalized);
+ return *this;
+ } else {
+ SplitModernName(topicNormalized);
+
+ BuildModernNames(TString(root));
+ if (supposeLegacyMode) {
+ ClientsideName = FullLegacyName;
+ PrimaryFullPath = FullLegacyPath;
+ } else {
+ ClientsideName = ModernName;
+ PrimaryFullPath = FullModernPath;
+ }
+ }
+ return *this;
+ }
+
+public:
+ bool IsValid() const {
+ return Valid;
+ }
+
+ const TString& GetReason() {
+ return Reason;
+ }
+
+ TString GetClientsideName() {
+ CHECK_VALID_AND_RETURN(ClientsideName);
+ }
+
+ TString GetPrimaryPath() {
+ CHECK_VALID_AND_RETURN(PrimaryFullPath);
+ }
+
+ TString GetSecondaryPath() {
+ Y_FAIL("UNIMPLEMENTED");
+ CHECK_VALID_AND_RETURN(SecondaryFullPath);
+ }
+
+ TString GetOriginalPath() const {
+ CHECK_VALID_AND_RETURN(OriginalTopic);
+ }
+
+ TString GetShortLegacyName() const {
+ CHECK_VALID_AND_RETURN(LegacyName);
+ }
+
+ TString GetFullLegacyName() const {
+ CHECK_VALID_AND_RETURN(FullLegacyName);
+ }
+
+ TString GetFullLegacyPath() const {
+ CHECK_VALID_AND_RETURN(FullLegacyPath);
+ }
+ TString GetLegacyProducer() const {
+ CHECK_VALID_AND_RETURN(Producer);
+ }
+
+ TString GetModernName() const {
+ CHECK_VALID_AND_RETURN(ModernName);
+ }
+
+ TString GetModernPath() const {
+ CHECK_VALID_AND_RETURN(FullModernPath);
+ }
+ TString GetCluster() const {
+ return Dc;
+ }
+
+ TString GetAccount() const {
+ if (!Valid)
+ return {};
+ if (!Database.empty()) {
+ return Database;
+ }
+ TStringBuf buf(ModernName);
+ TStringBuf fst, snd;
+ auto res = buf.TrySplit("/", fst, snd);
+ if (!res) {
+ fst = buf;
+ }
+ return TString{fst};
+ }
+
+private:
+ bool IsLegacyTopicName(const TStringBuf& path) {
+ if (path.find("--") != TStringBuf::npos)
+ return true;
+ if (Database.empty() && !NoDcMode && path.find("/") == TStringBuf::npos)
+ return true;
+ return false;
+ }
+
+ bool IsLegacyTopicNameWithDc(const TStringBuf& path) {
+ return path.StartsWith("rt3.");
+ }
+
+ void SplitFullLegacyName(TStringBuf& name) {
+ // rt3.sas--accountName[@dirName]--topicName
+ FullLegacyName = name;
+ TStringBuf fst, snd;
+ name.SkipPrefix("rt3.");
+ auto res = name.TrySplit("--", fst, snd);
+ CHECK_SET_VALID(res, Sprintf("topic name %s is corrupted", OriginalTopic.c_str()), return);
+ Dc = fst;
+ LegacyName = snd;
+ FullLegacyPath = NKikimr::JoinPath({TString(PQRootPrefix), FullLegacyName});
+ SplitShortLegacyName(snd);
+ }
+
+ void SplitShortLegacyName(const TStringBuf& name) {
+ if (LegacyName.empty())
+ LegacyName = name;
+ CHECK_SET_VALID(!NoDcMode, "legacy-style topic name as first-class citizen", return);
+ TStringBuf buf, logtype;
+ TStringBuilder topicPath;
+ auto res = name.TryRSplit("--", buf, logtype);
+ if (!res) {
+ topicPath << name;
+ Producer = name;
+ } else {
+ Producer = buf;
+ while (true) {
+ TStringBuf fst, snd;
+ if (!buf.TrySplit("@", fst, snd))
+ break;
+ topicPath << fst << "/";
+ buf = fst;
+ }
+ topicPath << buf << "/" << logtype;
+ }
+ ModernName = topicPath;
+ BuildFullLegacyName();
+ PrimaryFullPath = FullLegacyPath;
+ if (ClientsideName.empty() && !NoDcMode && !FullLegacyName.empty()) {
+ ClientsideName = FullLegacyName;
+ }
+ }
+
+ void SplitModernName(TStringBuf& name) {
+ if (ModernName.empty()) {
+ ModernName = name;
+ }
+ TStringBuf fst, snd;
+
+ auto res = name.TrySplit("/", fst, snd);
+ if (!res) {
+ LegacyName = name;
+ Producer = name;
+ } else {
+ TStringBuilder legacyName;
+ legacyName << fst;
+ while (true) {
+ name = snd;
+ res = name.TrySplit("/", fst, snd);
+ if (!res)
+ break;
+ legacyName << "@" << fst;
+ }
+ Producer = legacyName;
+ legacyName << "--" << name;
+ LegacyName = legacyName;
+ }
+ BuildFullLegacyName();
+ }
+
+ void BuildModernNames(const TString& prefix) {
+ if (!NoDcMode && !Dc.empty()) {
+ // ToDo[migration]: build full modern name, full modern path
+ } else {
+ FullModernName = ModernName;
+ FullModernPath = NKikimr::JoinPath({prefix, FullModernName});
+ }
+ }
+ void BuildFullLegacyName() {
+ if (FullLegacyName.empty() || !Dc.empty()) {
+ TStringBuilder builder;
+ builder << "rt3." << Dc << "--" << LegacyName;
+ FullLegacyName = builder;
+ }
+ if (FullLegacyPath.empty() && !FullLegacyName.empty()) {
+ FullLegacyPath = NKikimr::JoinPath({PQRootPrefix, FullLegacyName});
+ }
+ }
+
+
+ bool NoDcMode;
+ TString PQRootPrefix;
+
+ TString OriginalTopic;
+ TString Database;
+ TString Dc;
+
+ TString LegacyName;
+ TString FullLegacyName;
+ TString FullLegacyPath;
+
+ TString ModernName;
+ TString FullModernName;
+ TString FullModernPath;
+
+ TString PrimaryFullPath;
+ TString SecondaryFullPath;
+ TString ClientsideName;
+
+ TString Producer;
+ TString Account;
+ bool HasDc = false;
+
+ bool Valid = true;
+ TString Reason;
+};
+
+using TConverterPtr = std::shared_ptr<TTopicNameConverter>;
+
+class TTopicNamesConverterFactory {
+public:
+ TTopicNamesConverterFactory(bool noDcMode, const TString& pqRootPrefix)
+ : NoDcMode(noDcMode)
+ , PQRootPrefix(pqRootPrefix)
+ , NormalizedPrefix(PQRootPrefix)
+ {
+ NormalizedPrefix.SkipPrefix("/");
+ NormalizedPrefix.ChopSuffix("/");
+ }
+
+ TConverterPtr MakeTopicNameConverter(
+ const TString& topic, const TString& dc = TString(), const TString& database = TString()
+ ) {
+ auto* converter = new TTopicNameConverter(NoDcMode, NormalizedPrefix);
+ converter->ApplyToTopic(topic, dc, database);
+ return TConverterPtr(converter);
+ }
+
+private:
+ bool NoDcMode;
+ TString PQRootPrefix;
+ TStringBuf NormalizedPrefix;
+};
+
+using TConverterFactoryPtr = std::shared_ptr<NPersQueue::TTopicNamesConverterFactory>;
+using TTopicsToConverter = THashMap<TString, TConverterPtr>;
+
+class TTopicsListController {
+public:
+ TTopicsListController(const TConverterFactoryPtr& converterFactory,
+ bool haveClusters, const TVector<TString>& clusters = {}, const TString& localCluster = {});
+
+ void UpdateClusters(const TVector<TString>& clusters, const TString& localCluster);
+ TTopicsToConverter GetReadTopicsList(const THashSet<TString>& clientTopics, bool onlyLocal, const TString& database) const;
+ TConverterPtr GetWriteTopicConverter(const TString& clientName, const TString& database);
+ TConverterFactoryPtr GetConverterFactory() const;
+
+ bool GetHaveClusters() const { return HaveClusters;};
+
+private:
+ TConverterFactoryPtr ConverterFactory;
+ bool HaveClusters;
+ TVector<TString> Clusters;
+ TString LocalCluster;
+
+};
+
+TString NormalizeFullPath(const TString& fullPath);
+
+
} // namespace NPersQueue
-
+
diff --git a/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp b/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp
index a50bcf47c8..d684d43172 100644
--- a/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp
+++ b/ydb/library/persqueue/topic_parser/ut/topic_names_converter_ut.cpp
@@ -1,176 +1,176 @@
#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NPersQueue;
-namespace NTests {
-
-void BasicLegacyModeChecks(TTopicNameConverter* converter) {
- UNIT_ASSERT(converter->IsValid());
- UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), converter->GetFullLegacyPath());
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetFullLegacyName());
-}
-
-void BasicPathModeChecks(TTopicNameConverter* converter, const TString& fullPath) {
- UNIT_ASSERT_C(converter->IsValid(), converter->GetReason());
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetModernName());
- UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), fullPath);
- UNIT_ASSERT_VALUES_EQUAL(converter->GetModernPath(), fullPath);
-}
-
-Y_UNIT_TEST_SUITE(TopicNamesConverterTest) {
- Y_UNIT_TEST(LegacyModeWithMinimalName) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
-
- auto converter = factory.MakeTopicNameConverter("topic1", "dc1", "");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic1");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic1");
- BasicLegacyModeChecks(converter.get());
-
- converter = factory.MakeTopicNameConverter("topic2", "dc1", "");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic2");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic2");
- BasicLegacyModeChecks(converter.get());
- }
-
- Y_UNIT_TEST(LegacyModeWithReset) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
- auto converter = factory.MakeTopicNameConverter("topic1", "dc1", "");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1");
- BasicLegacyModeChecks(converter.get());
-
- converter = factory.MakeTopicNameConverter("topic1", "dc2", "");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic1");
- BasicLegacyModeChecks(converter.get());
-
- }
-
- Y_UNIT_TEST(LegacyModeWithShortNameAndDc) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
- auto converter = factory.MakeTopicNameConverter("rt3.dc2--topic2", "", "");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic2");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--topic2");
- BasicLegacyModeChecks(converter.get());
- }
-
- Y_UNIT_TEST(LegacyModeWithNormalNameAndRootDB) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
- auto converter = factory.MakeTopicNameConverter("account--topic1", "dc2", "/Root");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account--topic1");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--account--topic1");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--account--topic1");
- BasicLegacyModeChecks(converter.get());
-
- converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc3--account3--topic3", "", "");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account3--topic3");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc3--account3--topic3");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc3--account3--topic3");
- BasicLegacyModeChecks(converter.get());
- }
-
- Y_UNIT_TEST(LegacyModeWithFullNameAndRootDB) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
- auto converter = factory.MakeTopicNameConverter("rt3.sas--accountX@dir--topicX", "", "/Root");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "accountX@dir--topicX");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.sas--accountX@dir--topicX");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.sas--accountX@dir--topicX");
- BasicLegacyModeChecks(converter.get());
- }
-
- Y_UNIT_TEST(PathModeWithConversionToLegacy) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
- auto converter = factory.MakeTopicNameConverter("account1/topic1", "dc", "/Root");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account1--topic1");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc--account1--topic1");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc--account1--topic1");
- BasicLegacyModeChecks(converter.get());
- }
-
- Y_UNIT_TEST(LegacyModeWithErrors) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
- //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "")->IsValid());
- //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "/database")->IsValid());
- UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid());
- UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid());
- UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid());
-
- }
-
- Y_UNIT_TEST(PathWithNoDc) {
- TTopicNamesConverterFactory factory(false, "/Root/PQ");
- auto converter = factory.MakeTopicNameConverter("account/topic1", "", "");
- UNIT_ASSERT(converter->IsValid());
- UNIT_ASSERT_VALUES_EQUAL(converter->GetModernName(), "account/topic1");
- }
-
- Y_UNIT_TEST(FstClassModeWithShortTopic) {
- TTopicNamesConverterFactory factory(true, "");
- {
- auto converter = factory.MakeTopicNameConverter("topic1", "", "/database");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic1");
- BasicPathModeChecks(converter.get(), "database/topic1");
- }
- {
- auto converter = factory.MakeTopicNameConverter("topic2", "", "database2");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic2");
- BasicPathModeChecks(converter.get(), "database2/topic2");
- }
- {
- auto converter = factory.MakeTopicNameConverter("topic3", "", "database3/");
- BasicPathModeChecks(converter.get(), "database3/topic3");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic3");
- }
- }
- Y_UNIT_TEST(FstClassModeWithLegacyNames) {
- {
- TTopicNamesConverterFactory factory(true, "Root/PQ");
- auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root");
- UNIT_ASSERT(!converter->IsValid());
- converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "");
- UNIT_ASSERT(!converter->IsValid());
- }
- {
- TTopicNamesConverterFactory factory(true, "");
- auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root");
- UNIT_ASSERT(!converter->IsValid());
- }
- }
-
- Y_UNIT_TEST(FstClassModeWithRoot) {
- TVector<TTopicNamesConverterFactory> factories{
- {true, ""},
- {true, "/Root/PQ"}
- };
- for (auto& factory: factories) {
- {
- auto converter = factory.MakeTopicNameConverter("account/topic1", "", "/Root");
- BasicPathModeChecks(converter.get(), "Root/account/topic1");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account/topic1");
- }
- {
- auto converter = factory.MakeTopicNameConverter("/Root/account2/topic2", "", "/Root");
- BasicPathModeChecks(converter.get(), "Root/account2/topic2");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account2/topic2");
- }
- }
- }
-
- Y_UNIT_TEST(FstClassModeWithLongTopic) {
- TTopicNamesConverterFactory factory(true, "");
-
- auto converter = factory.MakeTopicNameConverter("dir4/topic4", "", "/database4");
- BasicPathModeChecks(converter.get(), "database4/dir4/topic4");
- UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "dir4/topic4");
-
- }
-
- Y_UNIT_TEST(FstClassModeWithBadTopics) {
- TTopicNamesConverterFactory factory(true, "/Root/PQ");
- UNIT_ASSERT(!factory.MakeTopicNameConverter("topic1", "", "")->IsValid());
- }
-
-}
-
-} // NTests \ No newline at end of file
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NPersQueue;
+namespace NTests {
+
+void BasicLegacyModeChecks(TTopicNameConverter* converter) {
+ UNIT_ASSERT(converter->IsValid());
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), converter->GetFullLegacyPath());
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetFullLegacyName());
+}
+
+void BasicPathModeChecks(TTopicNameConverter* converter, const TString& fullPath) {
+ UNIT_ASSERT_C(converter->IsValid(), converter->GetReason());
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), converter->GetModernName());
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetPrimaryPath(), fullPath);
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetModernPath(), fullPath);
+}
+
+Y_UNIT_TEST_SUITE(TopicNamesConverterTest) {
+ Y_UNIT_TEST(LegacyModeWithMinimalName) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+
+ auto converter = factory.MakeTopicNameConverter("topic1", "dc1", "");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic1");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic1");
+ BasicLegacyModeChecks(converter.get());
+
+ converter = factory.MakeTopicNameConverter("topic2", "dc1", "");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic2");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc1--topic2");
+ BasicLegacyModeChecks(converter.get());
+ }
+
+ Y_UNIT_TEST(LegacyModeWithReset) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+ auto converter = factory.MakeTopicNameConverter("topic1", "dc1", "");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc1--topic1");
+ BasicLegacyModeChecks(converter.get());
+
+ converter = factory.MakeTopicNameConverter("topic1", "dc2", "");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic1");
+ BasicLegacyModeChecks(converter.get());
+
+ }
+
+ Y_UNIT_TEST(LegacyModeWithShortNameAndDc) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+ auto converter = factory.MakeTopicNameConverter("rt3.dc2--topic2", "", "");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "topic2");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--topic2");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--topic2");
+ BasicLegacyModeChecks(converter.get());
+ }
+
+ Y_UNIT_TEST(LegacyModeWithNormalNameAndRootDB) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+ auto converter = factory.MakeTopicNameConverter("account--topic1", "dc2", "/Root");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account--topic1");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc2--account--topic1");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc2--account--topic1");
+ BasicLegacyModeChecks(converter.get());
+
+ converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc3--account3--topic3", "", "");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account3--topic3");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc3--account3--topic3");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc3--account3--topic3");
+ BasicLegacyModeChecks(converter.get());
+ }
+
+ Y_UNIT_TEST(LegacyModeWithFullNameAndRootDB) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+ auto converter = factory.MakeTopicNameConverter("rt3.sas--accountX@dir--topicX", "", "/Root");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "accountX@dir--topicX");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.sas--accountX@dir--topicX");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.sas--accountX@dir--topicX");
+ BasicLegacyModeChecks(converter.get());
+ }
+
+ Y_UNIT_TEST(PathModeWithConversionToLegacy) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+ auto converter = factory.MakeTopicNameConverter("account1/topic1", "dc", "/Root");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetShortLegacyName(), "account1--topic1");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyName(), "rt3.dc--account1--topic1");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetFullLegacyPath(), "Root/PQ/rt3.dc--account1--topic1");
+ BasicLegacyModeChecks(converter.get());
+ }
+
+ Y_UNIT_TEST(LegacyModeWithErrors) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+ //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "")->IsValid());
+ //UNIT_ASSERT(!factory.MakeTopicNameConverter("topic", "", "/database")->IsValid());
+ UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid());
+ UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid());
+ UNIT_ASSERT(!factory.MakeTopicNameConverter("rt3.man--account2--topic", "man", "")->IsValid());
+
+ }
+
+ Y_UNIT_TEST(PathWithNoDc) {
+ TTopicNamesConverterFactory factory(false, "/Root/PQ");
+ auto converter = factory.MakeTopicNameConverter("account/topic1", "", "");
+ UNIT_ASSERT(converter->IsValid());
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetModernName(), "account/topic1");
+ }
+
+ Y_UNIT_TEST(FstClassModeWithShortTopic) {
+ TTopicNamesConverterFactory factory(true, "");
+ {
+ auto converter = factory.MakeTopicNameConverter("topic1", "", "/database");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic1");
+ BasicPathModeChecks(converter.get(), "database/topic1");
+ }
+ {
+ auto converter = factory.MakeTopicNameConverter("topic2", "", "database2");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic2");
+ BasicPathModeChecks(converter.get(), "database2/topic2");
+ }
+ {
+ auto converter = factory.MakeTopicNameConverter("topic3", "", "database3/");
+ BasicPathModeChecks(converter.get(), "database3/topic3");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "topic3");
+ }
+ }
+ Y_UNIT_TEST(FstClassModeWithLegacyNames) {
+ {
+ TTopicNamesConverterFactory factory(true, "Root/PQ");
+ auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root");
+ UNIT_ASSERT(!converter->IsValid());
+ converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "");
+ UNIT_ASSERT(!converter->IsValid());
+ }
+ {
+ TTopicNamesConverterFactory factory(true, "");
+ auto converter = factory.MakeTopicNameConverter("/Root/PQ/rt3.dc1--topic3", "", "/Root");
+ UNIT_ASSERT(!converter->IsValid());
+ }
+ }
+
+ Y_UNIT_TEST(FstClassModeWithRoot) {
+ TVector<TTopicNamesConverterFactory> factories{
+ {true, ""},
+ {true, "/Root/PQ"}
+ };
+ for (auto& factory: factories) {
+ {
+ auto converter = factory.MakeTopicNameConverter("account/topic1", "", "/Root");
+ BasicPathModeChecks(converter.get(), "Root/account/topic1");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account/topic1");
+ }
+ {
+ auto converter = factory.MakeTopicNameConverter("/Root/account2/topic2", "", "/Root");
+ BasicPathModeChecks(converter.get(), "Root/account2/topic2");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "account2/topic2");
+ }
+ }
+ }
+
+ Y_UNIT_TEST(FstClassModeWithLongTopic) {
+ TTopicNamesConverterFactory factory(true, "");
+
+ auto converter = factory.MakeTopicNameConverter("dir4/topic4", "", "/database4");
+ BasicPathModeChecks(converter.get(), "database4/dir4/topic4");
+ UNIT_ASSERT_VALUES_EQUAL(converter->GetClientsideName(), "dir4/topic4");
+
+ }
+
+ Y_UNIT_TEST(FstClassModeWithBadTopics) {
+ TTopicNamesConverterFactory factory(true, "/Root/PQ");
+ UNIT_ASSERT(!factory.MakeTopicNameConverter("topic1", "", "")->IsValid());
+ }
+
+}
+
+} // NTests \ No newline at end of file
diff --git a/ydb/library/persqueue/topic_parser/ut/ya.make b/ydb/library/persqueue/topic_parser/ut/ya.make
index bfc6ca3074..74fdbf0bcd 100644
--- a/ydb/library/persqueue/topic_parser/ut/ya.make
+++ b/ydb/library/persqueue/topic_parser/ut/ya.make
@@ -1,24 +1,24 @@
UNITTEST_FOR(ydb/library/persqueue/topic_parser)
-
+
OWNER(
komels
g:kikimr
)
-
-FORK_SUBTESTS()
-TIMEOUT(30)
+FORK_SUBTESTS()
+
+TIMEOUT(30)
-SIZE(SMALL)
-
-PEERDIR(
- library/cpp/getopt
- library/cpp/svnversion
+SIZE(SMALL)
+
+PEERDIR(
+ library/cpp/getopt
+ library/cpp/svnversion
ydb/library/persqueue/topic_parser
-)
-
-SRCS(
- topic_names_converter_ut.cpp
-)
-
-END()
+)
+
+SRCS(
+ topic_names_converter_ut.cpp
+)
+
+END()
diff --git a/ydb/library/persqueue/topic_parser/ya.make b/ydb/library/persqueue/topic_parser/ya.make
index 0078e83431..2b4f126a59 100644
--- a/ydb/library/persqueue/topic_parser/ya.make
+++ b/ydb/library/persqueue/topic_parser/ya.make
@@ -1,5 +1,5 @@
OWNER(
- komels
+ komels
g:kikimr
g:logbroker
)
@@ -18,7 +18,7 @@ SRCS(
)
END()
-
+
RECURSE_FOR_TESTS(
- ut
-)
+ ut
+)
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp
index b939a1cfb7..dd42c8c4ed 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.cpp
@@ -2,8 +2,8 @@
#include <util/charset/unidata.h>
-namespace NYdb::NPersQueue {
-
+namespace NYdb::NPersQueue {
+
IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status) {
switch (status) {
case EStatus::SUCCESS:
@@ -29,7 +29,7 @@ IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status) {
case EStatus::CLIENT_LIMITS_REACHED:
case EStatus::CLIENT_DISCOVERY_FAILED:
return IRetryPolicy::ERetryErrorClass::LongRetry;
-
+
case EStatus::SCHEME_ERROR:
case EStatus::STATUS_UNDEFINED:
case EStatus::BAD_REQUEST:
@@ -41,38 +41,38 @@ IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status) {
case EStatus::CLIENT_UNAUTHENTICATED:
case EStatus::CLIENT_CALL_UNIMPLEMENTED:
return IRetryPolicy::ERetryErrorClass::NoRetry;
- }
-}
-
-IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status) {
- switch (status) {
- case EStatus::SCHEME_ERROR:
- return IRetryPolicy::ERetryErrorClass::NoRetry;
- default:
- return GetRetryErrorClass(status);
-
- }
-}
-
-void Cancel(NGrpc::IQueueClientContextPtr& context) {
- if (context) {
- context->Cancel();
- }
-}
-
-NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues) {
- NYql::TIssues issues;
- NYql::TIssue issue(description);
- for (const NYql::TIssue& i : subissues) {
- issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i));
- }
- issues.AddIssue(std::move(issue));
- return issues;
-}
-
-size_t CalcDataSize(const TReadSessionEvent::TEvent& event) {
+ }
+}
+
+IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status) {
+ switch (status) {
+ case EStatus::SCHEME_ERROR:
+ return IRetryPolicy::ERetryErrorClass::NoRetry;
+ default:
+ return GetRetryErrorClass(status);
+
+ }
+}
+
+void Cancel(NGrpc::IQueueClientContextPtr& context) {
+ if (context) {
+ context->Cancel();
+ }
+}
+
+NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues) {
+ NYql::TIssues issues;
+ NYql::TIssue issue(description);
+ for (const NYql::TIssue& i : subissues) {
+ issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i));
+ }
+ issues.AddIssue(std::move(issue));
+ return issues;
+}
+
+size_t CalcDataSize(const TReadSessionEvent::TEvent& event) {
if (const TReadSessionEvent::TDataReceivedEvent* dataEvent = std::get_if<TReadSessionEvent::TDataReceivedEvent>(&event)) {
- size_t len = 0;
+ size_t len = 0;
if (dataEvent->IsCompressedMessages()) {
for (const auto& msg : dataEvent->GetCompressedMessages()) {
len += msg.GetData().size();
@@ -83,13 +83,13 @@ size_t CalcDataSize(const TReadSessionEvent::TEvent& event) {
len += msg.GetData().size();
}
}
- }
- return len;
- } else {
- return 0;
- }
-}
-
+ }
+ return len;
+ } else {
+ return 0;
+ }
+}
+
static TStringBuf SplitPort(TStringBuf endpoint) {
for (int i = endpoint.Size() - 1; i >= 0; --i) {
if (endpoint[i] == ':') {
@@ -121,90 +121,90 @@ TString ApplyClusterEndpoint(TStringBuf driverEndpoint, const TString& clusterDi
}
}
-void IAsyncExecutor::Post(TFunction&& f) {
- PostImpl(std::move(f));
-}
-
-IAsyncExecutor::TPtr CreateDefaultExecutor() {
+void IAsyncExecutor::Post(TFunction&& f) {
+ PostImpl(std::move(f));
+}
+
+IAsyncExecutor::TPtr CreateDefaultExecutor() {
return CreateThreadPoolExecutor(1);
-}
-
-void TThreadPoolExecutor::PostImpl(TVector<TFunction>&& fs) {
- for (auto& f : fs) {
- ThreadPool->SafeAddFunc(std::move(f));
- }
-}
-
-void TThreadPoolExecutor::PostImpl(TFunction&& f) {
- ThreadPool->SafeAddFunc(std::move(f));
-}
-
-TSerialExecutor::TSerialExecutor(IAsyncExecutor::TPtr executor)
+}
+
+void TThreadPoolExecutor::PostImpl(TVector<TFunction>&& fs) {
+ for (auto& f : fs) {
+ ThreadPool->SafeAddFunc(std::move(f));
+ }
+}
+
+void TThreadPoolExecutor::PostImpl(TFunction&& f) {
+ ThreadPool->SafeAddFunc(std::move(f));
+}
+
+TSerialExecutor::TSerialExecutor(IAsyncExecutor::TPtr executor)
: Executor(executor)
{
- Y_VERIFY(executor);
-}
-
-void TSerialExecutor::PostImpl(TVector<TFunction>&& fs) {
- for (auto& f : fs) {
- PostImpl(std::move(f));
- }
-}
-
-void TSerialExecutor::PostImpl(TFunction&& f) {
- with_lock(Mutex) {
- ExecutionQueue.push(std::move(f));
- if (Busy) {
- return;
- }
- PostNext();
- }
-}
-
-void TSerialExecutor::PostNext() {
- Y_VERIFY(!Busy);
-
- if (ExecutionQueue.empty()) {
- return;
- }
-
- auto weakThis = weak_from_this();
- Executor->Post([weakThis, f = std::move(ExecutionQueue.front())]() {
- if (auto sharedThis = weakThis.lock()) {
- f();
- with_lock(sharedThis->Mutex) {
- sharedThis->Busy = false;
- sharedThis->PostNext();
- }
- }
- });
- ExecutionQueue.pop();
- Busy = true;
-}
+ Y_VERIFY(executor);
+}
+
+void TSerialExecutor::PostImpl(TVector<TFunction>&& fs) {
+ for (auto& f : fs) {
+ PostImpl(std::move(f));
+ }
+}
+
+void TSerialExecutor::PostImpl(TFunction&& f) {
+ with_lock(Mutex) {
+ ExecutionQueue.push(std::move(f));
+ if (Busy) {
+ return;
+ }
+ PostNext();
+ }
+}
+
+void TSerialExecutor::PostNext() {
+ Y_VERIFY(!Busy);
+
+ if (ExecutionQueue.empty()) {
+ return;
+ }
+
+ auto weakThis = weak_from_this();
+ Executor->Post([weakThis, f = std::move(ExecutionQueue.front())]() {
+ if (auto sharedThis = weakThis.lock()) {
+ f();
+ with_lock(sharedThis->Mutex) {
+ sharedThis->Busy = false;
+ sharedThis->PostNext();
+ }
+ }
+ });
+ ExecutionQueue.pop();
+ Busy = true;
+}
IExecutor::TPtr CreateThreadPoolExecutor(size_t threads) {
return MakeIntrusive<TThreadPoolExecutor>(threads);
-}
-
+}
+
IExecutor::TPtr CreateGenericExecutor() {
return CreateThreadPoolExecutor(1);
-}
+}
IExecutor::TPtr CreateThreadPoolExecutorAdapter(std::shared_ptr<IThreadPool> threadPool) {
return MakeIntrusive<TThreadPoolExecutor>(std::move(threadPool));
-}
-
+}
+
TThreadPoolExecutor::TThreadPoolExecutor(std::shared_ptr<IThreadPool> threadPool)
: ThreadPool(std::move(threadPool))
{
IsFakeThreadPool = dynamic_cast<TFakeThreadPool*>(ThreadPool.get()) != nullptr;
-}
-
+}
+
TThreadPoolExecutor::TThreadPoolExecutor(size_t threadsCount)
: TThreadPoolExecutor(CreateThreadPool(threadsCount))
{
Y_VERIFY(threadsCount > 0);
ThreadsCount = threadsCount;
-}
+}
}
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h
index 96b671dee8..7287b69894 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h
@@ -5,36 +5,36 @@
#include <util/generic/queue.h>
#include <util/system/condvar.h>
-#include <util/thread/pool.h>
+#include <util/thread/pool.h>
#include <queue>
namespace NYdb::NPersQueue {
IRetryPolicy::ERetryErrorClass GetRetryErrorClass(EStatus status);
-IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status);
-
-void Cancel(NGrpc::IQueueClientContextPtr& context);
-
-NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues);
-
+IRetryPolicy::ERetryErrorClass GetRetryErrorClassV2(EStatus status);
+
+void Cancel(NGrpc::IQueueClientContextPtr& context);
+
+NYql::TIssues MakeIssueWithSubIssues(const TString& description, const NYql::TIssues& subissues);
+
TString IssuesSingleLineString(const NYql::TIssues& issues);
size_t CalcDataSize(const TReadSessionEvent::TEvent& event);
-template <class TMessage>
-bool IsErrorMessage(const TMessage& serverMessage) {
- const Ydb::StatusIds::StatusCode status = serverMessage.status();
- return status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::STATUS_CODE_UNSPECIFIED;
-}
-
-template <class TMessage>
-TPlainStatus MakeErrorFromProto(const TMessage& serverMessage) {
- NYql::TIssues issues;
- NYql::IssuesFromMessage(serverMessage.issues(), issues);
- return TPlainStatus(static_cast<EStatus>(serverMessage.status()), std::move(issues));
-}
-
+template <class TMessage>
+bool IsErrorMessage(const TMessage& serverMessage) {
+ const Ydb::StatusIds::StatusCode status = serverMessage.status();
+ return status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::STATUS_CODE_UNSPECIFIED;
+}
+
+template <class TMessage>
+TPlainStatus MakeErrorFromProto(const TMessage& serverMessage) {
+ NYql::TIssues issues;
+ NYql::IssuesFromMessage(serverMessage.issues(), issues);
+ return TPlainStatus(static_cast<EStatus>(serverMessage.status()), std::move(issues));
+}
+
// Gets source endpoint for the whole driver (or persqueue client)
// and endpoint that was given us by the cluster discovery service
// and gives endpoint for the current LB cluster.
@@ -189,22 +189,22 @@ std::shared_ptr<ISessionConnectionProcessorFactory<TRequest, TResponse>>
template <class TEvent_>
struct TBaseEventInfo {
using TEvent = TEvent_;
-
+
TEvent Event;
-
+
TEvent& GetEvent() {
return Event;
- }
-
+ }
+
void OnUserRetrievedEvent() {
- }
-
+ }
+
template <class T>
TBaseEventInfo(T&& event)
: Event(std::forward<T>(event))
{}
};
-
+
class ISignalable {
public:
@@ -278,27 +278,27 @@ protected:
using TSettings = TSettings_;
using TEvent = TEvent_;
using TEventInfo = TEventInfo_;
-
-
+
+
// Template for visitor implementation.
struct TBaseHandlersVisitor {
TBaseHandlersVisitor(const TSettings& settings, TEventInfo& eventInfo)
: Settings(settings)
, EventInfo(eventInfo)
- {}
-
+ {}
+
template <class TEventType, class TFunc, class TCommonFunc>
bool PushHandler(TEventInfo&& eventInfo, const TFunc& specific, const TCommonFunc& common) {
if (specific) {
PushSpecificHandler<TEventType>(std::move(eventInfo), specific);
return true;
- }
+ }
if (common) {
PushCommonHandler(std::move(eventInfo), common);
return true;
}
return false;
- }
+ }
template <class TEventType, class TFunc>
void PushSpecificHandler(TEventInfo&& eventInfo, const TFunc& f) {
@@ -320,17 +320,17 @@ protected:
executor->Post(std::move(f));
}
- const TSettings& Settings;
- TEventInfo& EventInfo;
- };
-
+ const TSettings& Settings;
+ TEventInfo& EventInfo;
+ };
+
public:
TBaseSessionEventsQueue(const TSettings& settings)
: Settings(settings)
, Waiter(NThreading::NewPromise<void>(), this)
{}
-
+
virtual ~TBaseSessionEventsQueue() = default;
@@ -338,11 +338,11 @@ public:
CondVar.Signal();
}
-protected:
+protected:
virtual bool HasEventsImpl() const { // Assumes that we're under lock.
- return !Events.empty() || CloseEvent;
- }
-
+ return !Events.empty() || CloseEvent;
+ }
+
TWaiter PopWaiterImpl() { // Assumes that we're under lock.
TWaiter waiter(Waiter.ExtractPromise(), this);
return std::move(waiter);
@@ -353,7 +353,7 @@ protected:
CondVar.WaitI(Mutex);
}
}
-
+
void RenewWaiterImpl() {
if (Events.empty() && Waiter.GetFuture().HasValue()) {
Waiter = TWaiter(NThreading::NewPromise<void>(), this);
@@ -373,41 +373,41 @@ public:
}
}
-protected:
- const TSettings& Settings;
+protected:
+ const TSettings& Settings;
TWaiter Waiter;
- std::queue<TEventInfo> Events;
- TCondVar CondVar;
- TMutex Mutex;
- TMaybe<TSessionClosedEvent> CloseEvent;
- std::atomic<bool> Closed = false;
-};
-
+ std::queue<TEventInfo> Events;
+ TCondVar CondVar;
+ TMutex Mutex;
+ TMaybe<TSessionClosedEvent> CloseEvent;
+ std::atomic<bool> Closed = false;
+};
+
class IAsyncExecutor : public IExecutor {
-private:
- virtual void PostImpl(TVector<std::function<void()>>&&) = 0;
- virtual void PostImpl(std::function<void()>&&) = 0;
-
-public:
+private:
+ virtual void PostImpl(TVector<std::function<void()>>&&) = 0;
+ virtual void PostImpl(std::function<void()>&&) = 0;
+
+public:
bool IsAsync() const override {
- return true;
- }
- // Post Implementation MUST NOT run f before it returns
- void Post(TFunction&& f) final;
-};
-
+ return true;
+ }
+ // Post Implementation MUST NOT run f before it returns
+ void Post(TFunction&& f) final;
+};
+
IExecutor::TPtr CreateDefaultExecutor();
-
-
-class TThreadPoolExecutor : public IAsyncExecutor {
-private:
+
+
+class TThreadPoolExecutor : public IAsyncExecutor {
+private:
std::shared_ptr<IThreadPool> ThreadPool;
-
-public:
+
+public:
TThreadPoolExecutor(std::shared_ptr<IThreadPool> threadPool);
TThreadPoolExecutor(size_t threadsCount);
- ~TThreadPoolExecutor() = default;
-
+ ~TThreadPoolExecutor() = default;
+
bool IsAsync() const override {
return !IsFakeThreadPool;
}
@@ -418,44 +418,44 @@ public:
}
}
-private:
- void PostImpl(TVector<TFunction>&& fs) override;
- void PostImpl(TFunction&& f) override;
+private:
+ void PostImpl(TVector<TFunction>&& fs) override;
+ void PostImpl(TFunction&& f) override;
private:
bool IsFakeThreadPool = false;
size_t ThreadsCount = 0;
-};
-
-class TSerialExecutor : public IAsyncExecutor, public std::enable_shared_from_this<TSerialExecutor> {
-private:
- IAsyncExecutor::TPtr Executor; //!< Wrapped executor that is actually doing the job
- bool Busy = false; //!< Set if some closure was scheduled for execution and did not finish yet
- TMutex Mutex = {};
- TQueue<TFunction> ExecutionQueue = {};
-
-public:
- TSerialExecutor(IAsyncExecutor::TPtr executor);
- ~TSerialExecutor() = default;
-
-private:
- void PostImpl(TVector<TFunction>&& fs) override;
- void PostImpl(TFunction&& f) override;
- void PostNext();
-};
-
+};
+
+class TSerialExecutor : public IAsyncExecutor, public std::enable_shared_from_this<TSerialExecutor> {
+private:
+ IAsyncExecutor::TPtr Executor; //!< Wrapped executor that is actually doing the job
+ bool Busy = false; //!< Set if some closure was scheduled for execution and did not finish yet
+ TMutex Mutex = {};
+ TQueue<TFunction> ExecutionQueue = {};
+
+public:
+ TSerialExecutor(IAsyncExecutor::TPtr executor);
+ ~TSerialExecutor() = default;
+
+private:
+ void PostImpl(TVector<TFunction>&& fs) override;
+ void PostImpl(TFunction&& f) override;
+ void PostNext();
+};
+
class TSyncExecutor : public IExecutor {
-public:
- void Post(TFunction&& f) final {
- return f();
- }
+public:
+ void Post(TFunction&& f) final {
+ return f();
+ }
bool IsAsync() const final {
- return false;
- }
+ return false;
+ }
void DoStart() override {
}
-};
-
+};
+
IExecutor::TPtr CreateGenericExecutor();
-
+
} // namespace NYdb::NPersQueue
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp
index 673d4bc7fb..a22ed23cea 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue.cpp
@@ -380,13 +380,13 @@ std::shared_ptr<IReadSession> TPersQueueClient::CreateReadSession(const TReadSes
}
std::shared_ptr<IWriteSession> TPersQueueClient::CreateWriteSession(const TWriteSessionSettings& settings) {
- return Impl_->CreateWriteSession(settings);
-}
+ return Impl_->CreateWriteSession(settings);
+}
std::shared_ptr<ISimpleBlockingWriteSession> TPersQueueClient::CreateSimpleBlockingWriteSession(
- const TWriteSessionSettings& settings
-) {
- return Impl_->CreateSimpleWriteSession(settings);
-}
+ const TWriteSessionSettings& settings
+) {
+ return Impl_->CreateSimpleWriteSession(settings);
+}
} // namespace NYdb::NPersQueue
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp
index 9541e92152..bd466fd6f3 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.cpp
@@ -23,8 +23,8 @@ std::shared_ptr<IReadSession> TPersQueueClient::TImpl::CreateReadSession(const T
}
std::shared_ptr<IWriteSession> TPersQueueClient::TImpl::CreateWriteSession(
- const TWriteSessionSettings& settings
-) {
+ const TWriteSessionSettings& settings
+) {
TMaybe<TWriteSessionSettings> maybeSettings;
if (!settings.CompressionExecutor_ || !settings.EventHandlers_.HandlersExecutor_ || !settings.ClusterDiscoveryMode_) {
maybeSettings = settings;
@@ -40,30 +40,30 @@ std::shared_ptr<IWriteSession> TPersQueueClient::TImpl::CreateWriteSession(
}
}
}
- auto session = std::make_shared<TWriteSession>(
+ auto session = std::make_shared<TWriteSession>(
maybeSettings.GetOrElse(settings), shared_from_this(), Connections_, DbDriverState_
- );
- session->Start(TDuration::Zero());
- return std::move(session);
-}
-
+ );
+ session->Start(TDuration::Zero());
+ return std::move(session);
+}
+
std::shared_ptr<ISimpleBlockingWriteSession> TPersQueueClient::TImpl::CreateSimpleWriteSession(
- const TWriteSessionSettings& settings
-) {
- auto alteredSettings = settings;
- with_lock (Lock) {
- alteredSettings.EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_);
- if (!settings.CompressionExecutor_) {
- alteredSettings.CompressionExecutor(Settings.DefaultCompressionExecutor_);
- }
- }
-
- auto session = std::make_shared<TSimpleBlockingWriteSession>(
+ const TWriteSessionSettings& settings
+) {
+ auto alteredSettings = settings;
+ with_lock (Lock) {
+ alteredSettings.EventHandlers_.HandlersExecutor(Settings.DefaultHandlersExecutor_);
+ if (!settings.CompressionExecutor_) {
+ alteredSettings.CompressionExecutor(Settings.DefaultCompressionExecutor_);
+ }
+ }
+
+ auto session = std::make_shared<TSimpleBlockingWriteSession>(
alteredSettings, shared_from_this(), Connections_, DbDriverState_
- );
- return std::move(session);
-}
-
+ );
+ return std::move(session);
+}
+
std::shared_ptr<TPersQueueClient::TImpl> TPersQueueClient::TImpl::GetClientForEndpoint(const TString& clusterEndoint) {
with_lock (Lock) {
Y_VERIFY(!CustomEndpoint);
@@ -83,10 +83,10 @@ std::shared_ptr<TPersQueueClient::TImpl::IReadSessionConnectionProcessorFactory>
}
std::shared_ptr<TPersQueueClient::TImpl::IWriteSessionConnectionProcessorFactory> TPersQueueClient::TImpl::CreateWriteSessionConnectionProcessorFactory() {
- using TService = Ydb::PersQueue::V1::PersQueueService;
- using TRequest = Ydb::PersQueue::V1::StreamingWriteClientMessage;
- using TResponse = Ydb::PersQueue::V1::StreamingWriteServerMessage;
- return CreateConnectionProcessorFactory<TService, TRequest, TResponse>(&TService::Stub::AsyncStreamingWrite, Connections_, DbDriverState_);
-}
-
+ using TService = Ydb::PersQueue::V1::PersQueueService;
+ using TRequest = Ydb::PersQueue::V1::StreamingWriteClientMessage;
+ using TResponse = Ydb::PersQueue::V1::StreamingWriteServerMessage;
+ return CreateConnectionProcessorFactory<TService, TRequest, TResponse>(&TService::Stub::AsyncStreamingWrite, Connections_, DbDriverState_);
+}
+
} // namespace NYdb::NPersQueue
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h
index f97469ac96..57e9dd43bf 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/persqueue_impl.h
@@ -195,8 +195,8 @@ public:
// Runtime API.
std::shared_ptr<IReadSession> CreateReadSession(const TReadSessionSettings& settings);
- std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleWriteSession(const TWriteSessionSettings& settings);
- std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings);
+ std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleWriteSession(const TWriteSessionSettings& settings);
+ std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings);
std::shared_ptr<TImpl> GetClientForEndpoint(const TString& clusterEndoint);
@@ -204,12 +204,12 @@ public:
std::shared_ptr<IReadSessionConnectionProcessorFactory> CreateReadSessionConnectionProcessorFactory();
- using IWriteSessionConnectionProcessorFactory = ISessionConnectionProcessorFactory<
- Ydb::PersQueue::V1::StreamingWriteClientMessage,
- Ydb::PersQueue::V1::StreamingWriteServerMessage>;
-
- std::shared_ptr<IWriteSessionConnectionProcessorFactory> CreateWriteSessionConnectionProcessorFactory();
-
+ using IWriteSessionConnectionProcessorFactory = ISessionConnectionProcessorFactory<
+ Ydb::PersQueue::V1::StreamingWriteClientMessage,
+ Ydb::PersQueue::V1::StreamingWriteServerMessage>;
+
+ std::shared_ptr<IWriteSessionConnectionProcessorFactory> CreateWriteSessionConnectionProcessorFactory();
+
NGrpc::IQueueClientContextPtr CreateContext() {
return Connections_->CreateContext();
}
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h
index 7d77159d75..b5b874954f 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/read_session.h
@@ -103,7 +103,7 @@ public:
std::weak_ptr<TSingleClusterReadSessionImpl> session,
bool doDecompress
);
-
+
i64 StartDecompressionTasks(const IExecutor::TPtr& executor,
i64 availableMemory,
double averageCompressionRatio,
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp
index a469a2a8be..af4e08b979 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.cpp
@@ -4,354 +4,354 @@
#include <util/generic/store_policy.h>
#include <util/generic/utility.h>
-#include <util/stream/buffer.h>
-
+#include <util/stream/buffer.h>
+
+
+namespace NYdb::NPersQueue {
+using NMonitoring::TDynamicCounterPtr;
+using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
-namespace NYdb::NPersQueue {
-using NMonitoring::TDynamicCounterPtr;
-using TCounterPtr = NMonitoring::TDynamicCounters::TCounterPtr;
-
const TDuration UPDATE_TOKEN_PERIOD = TDuration::Hours(1);
namespace NCompressionDetails {
THolder<IOutputStream> CreateCoder(ECodec codec, TBuffer& result, int quality);
-}
-
-#define HISTOGRAM_SETUP NMonitoring::ExplicitHistogram({0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100})
+}
+
+#define HISTOGRAM_SETUP NMonitoring::ExplicitHistogram({0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100})
TWriterCounters::TWriterCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters) {
- Errors = counters->GetCounter("errors", true);
- CurrentSessionLifetimeMs = counters->GetCounter("currentSessionLifetimeMs", false);
- BytesWritten = counters->GetCounter("bytesWritten", true);
- MessagesWritten = counters->GetCounter("messagesWritten", true);
- BytesWrittenCompressed = counters->GetCounter("bytesWrittenCompressed", true);
- BytesInflightUncompressed = counters->GetCounter("bytesInflightUncompressed", false);
- BytesInflightCompressed = counters->GetCounter("bytesInflightCompressed", false);
- BytesInflightTotal = counters->GetCounter("bytesInflightTotal", false);
- MessagesInflight = counters->GetCounter("messagesInflight", false);
-
- TotalBytesInflightUsageByTime = counters->GetHistogram("totalBytesInflightUsageByTime", HISTOGRAM_SETUP);
- UncompressedBytesInflightUsageByTime = counters->GetHistogram("uncompressedBytesInflightUsageByTime", HISTOGRAM_SETUP);
- CompressedBytesInflightUsageByTime = counters->GetHistogram("compressedBytesInflightUsageByTime", HISTOGRAM_SETUP);
-}
-#undef HISTOGRAM_SETUP
-
-TWriteSession::TWriteSession(
- const TWriteSessionSettings& settings,
+ Errors = counters->GetCounter("errors", true);
+ CurrentSessionLifetimeMs = counters->GetCounter("currentSessionLifetimeMs", false);
+ BytesWritten = counters->GetCounter("bytesWritten", true);
+ MessagesWritten = counters->GetCounter("messagesWritten", true);
+ BytesWrittenCompressed = counters->GetCounter("bytesWrittenCompressed", true);
+ BytesInflightUncompressed = counters->GetCounter("bytesInflightUncompressed", false);
+ BytesInflightCompressed = counters->GetCounter("bytesInflightCompressed", false);
+ BytesInflightTotal = counters->GetCounter("bytesInflightTotal", false);
+ MessagesInflight = counters->GetCounter("messagesInflight", false);
+
+ TotalBytesInflightUsageByTime = counters->GetHistogram("totalBytesInflightUsageByTime", HISTOGRAM_SETUP);
+ UncompressedBytesInflightUsageByTime = counters->GetHistogram("uncompressedBytesInflightUsageByTime", HISTOGRAM_SETUP);
+ CompressedBytesInflightUsageByTime = counters->GetHistogram("compressedBytesInflightUsageByTime", HISTOGRAM_SETUP);
+}
+#undef HISTOGRAM_SETUP
+
+TWriteSession::TWriteSession(
+ const TWriteSessionSettings& settings,
std::shared_ptr<TPersQueueClient::TImpl> client,
- std::shared_ptr<TGRpcConnectionsImpl> connections,
- TDbDriverStatePtr dbDriverState)
- : Settings(settings)
- , Client(std::move(client))
- , Connections(std::move(connections))
- , DbDriverState(std::move(dbDriverState))
+ std::shared_ptr<TGRpcConnectionsImpl> connections,
+ TDbDriverStatePtr dbDriverState)
+ : Settings(settings)
+ , Client(std::move(client))
+ , Connections(std::move(connections))
+ , DbDriverState(std::move(dbDriverState))
, PrevToken(DbDriverState->CredentialsProvider ? DbDriverState->CredentialsProvider->GetAuthInfo() : "")
, EventsQueue(std::make_shared<TWriteSessionEventsQueue>(Settings))
, InitSeqNoPromise(NThreading::NewPromise<ui64>())
- , WakeupInterval(
- Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero()) ?
- std::min(Settings.BatchFlushInterval_.GetOrElse(TDuration::Seconds(1)) / 5, TDuration::MilliSeconds(100))
- :
- TDuration::MilliSeconds(100)
- )
-{
- if (!Settings.RetryPolicy_) {
- Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy();
- }
+ , WakeupInterval(
+ Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero()) ?
+ std::min(Settings.BatchFlushInterval_.GetOrElse(TDuration::Seconds(1)) / 5, TDuration::MilliSeconds(100))
+ :
+ TDuration::MilliSeconds(100)
+ )
+{
+ if (!Settings.RetryPolicy_) {
+ Settings.RetryPolicy_ = IRetryPolicy::GetDefaultPolicy();
+ }
if (Settings.PreferredCluster_ && !Settings.AllowFallbackToOtherClusters_) {
TargetCluster = *Settings.PreferredCluster_;
- TargetCluster.to_lower();
- }
- if (Settings.Counters_.Defined()) {
- Counters = *Settings.Counters_;
- } else {
- Counters = MakeIntrusive<TWriterCounters>(new NMonitoring::TDynamicCounters());
- }
-
-}
-
-void TWriteSession::Start(const TDuration& delay) {
- ++ConnectionAttemptsDone;
- if (!Started) {
- HandleWakeUpImpl();
- InitWriter();
- }
- Started = true;
+ TargetCluster.to_lower();
+ }
+ if (Settings.Counters_.Defined()) {
+ Counters = *Settings.Counters_;
+ } else {
+ Counters = MakeIntrusive<TWriterCounters>(new NMonitoring::TDynamicCounters());
+ }
+
+}
+
+void TWriteSession::Start(const TDuration& delay) {
+ ++ConnectionAttemptsDone;
+ if (!Started) {
+ HandleWakeUpImpl();
+ InitWriter();
+ }
+ Started = true;
DoCdsRequest(delay);
-}
-
-// Only called under lock
-TWriteSession::THandleResult TWriteSession::RestartImpl(const TPlainStatus& status) {
- THandleResult result;
- if (AtomicGet(Aborting)) {
+}
+
+// Only called under lock
+TWriteSession::THandleResult TWriteSession::RestartImpl(const TPlainStatus& status) {
+ THandleResult result;
+ if (AtomicGet(Aborting)) {
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session is aborting and will not restart";
- return result;
- }
+ return result;
+ }
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Got error. Status: " << status.Status << ". Description: " << IssuesSingleLineString(status.Issues);
- SessionEstablished = false;
- TMaybe<TDuration> nextDelay = TDuration::Zero();
- if (!RetryState) {
- RetryState = Settings.RetryPolicy_->CreateRetryState();
- }
+ SessionEstablished = false;
+ TMaybe<TDuration> nextDelay = TDuration::Zero();
+ if (!RetryState) {
+ RetryState = Settings.RetryPolicy_->CreateRetryState();
+ }
nextDelay = RetryState->GetNextRetryDelay(TPlainStatus(status));
-
- if (nextDelay) {
- result.StartDelay = *nextDelay;
- result.DoRestart = true;
+
+ if (nextDelay) {
+ result.StartDelay = *nextDelay;
+ result.DoRestart = true;
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session will restart in " << result.StartDelay.MilliSeconds() << " ms";
- ResetForRetryImpl();
-
- } else {
+ ResetForRetryImpl();
+
+ } else {
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will not restart after a fatal error";
- result.DoStop = true;
- CheckHandleResultImpl(result);
- }
- return result;
-}
-
+ result.DoStop = true;
+ CheckHandleResultImpl(result);
+ }
+ return result;
+}
+
bool IsFederation(const TString& endpoint) {
TStringBuf host = GetHost(endpoint);
return host == "logbroker.yandex.net" || host == "logbroker-prestable.yandex.net";
}
-void TWriteSession::DoCdsRequest(TDuration delay) {
+void TWriteSession::DoCdsRequest(TDuration delay) {
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session: Do CDS request";
auto weakThis = weak_from_this();
-
- if (
- Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Off ||
- (Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Auto && !IsFederation(DbDriverState->DiscoveryEndpoint))
- ) {
+
+ if (
+ Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Off ||
+ (Settings.ClusterDiscoveryMode_ == EClusterDiscoveryMode::Auto && !IsFederation(DbDriverState->DiscoveryEndpoint))
+ ) {
DoConnect(delay, DbDriverState->DiscoveryEndpoint);
return;
}
auto extractor = [weakThis]
- (google::protobuf::Any* any, TPlainStatus status) mutable {
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result;
- if (any) {
- any->UnpackTo(&result);
- }
+ (google::protobuf::Any* any, TPlainStatus status) mutable {
+ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result;
+ if (any) {
+ any->UnpackTo(&result);
+ }
TStatus st(std::move(status));
if (auto sharedThis = weakThis.lock()) {
sharedThis->OnCdsResponse(st, result);
}
- };
-
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req;
- auto* params = req.add_write_sessions();
- params->set_topic(Settings.Path_);
- params->set_source_id(Settings.MessageGroupId_);
- if (Settings.PartitionGroupId_.Defined())
- params->set_partition_group(*Settings.PartitionGroupId_);
+ };
+
+ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req;
+ auto* params = req.add_write_sessions();
+ params->set_topic(Settings.Path_);
+ params->set_source_id(Settings.MessageGroupId_);
+ if (Settings.PartitionGroupId_.Defined())
+ params->set_partition_group(*Settings.PartitionGroupId_);
if (Settings.PreferredCluster_.Defined())
params->set_preferred_cluster_name(*Settings.PreferredCluster_);
-
+
auto weakConnections = std::weak_ptr<TGRpcConnectionsImpl>(Connections);
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Do schedule cds request after " << delay.MilliSeconds() << " ms\n";
auto cdsRequestCall = [req_=std::move(req), extr=std::move(extractor), weakConnections, dbState=DbDriverState, settings=Settings]() mutable {
if (auto connections = weakConnections.lock()) {
dbState->Log << TLOG_INFO << "MessageGroupId [" << settings.MessageGroupId_ << "] Running cds request ms\n";
connections->RunDeferred<Ydb::PersQueue::V1::ClusterDiscoveryService,
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest,
- Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse>(
- std::move(req_),
- std::move(extr),
- &Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters,
- dbState,
- INITIAL_DEFERRED_CALL_DELAY,
- TRpcRequestSettings::Make(settings),
+ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest,
+ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse>(
+ std::move(req_),
+ std::move(extr),
+ &Ydb::PersQueue::V1::ClusterDiscoveryService::Stub::AsyncDiscoverClusters,
+ dbState,
+ INITIAL_DEFERRED_CALL_DELAY,
+ TRpcRequestSettings::Make(settings),
settings.ConnectTimeout_); // TODO: make client timeout setting
}
- };
- Connections->ScheduleOneTimeTask(std::move(cdsRequestCall), delay);
-}
-
-void TWriteSession::OnCdsResponse(
- TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result
-) {
+ };
+ Connections->ScheduleOneTimeTask(std::move(cdsRequestCall), delay);
+}
+
+void TWriteSession::OnCdsResponse(
+ TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result
+) {
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Got CDS response: \n" << result.ShortDebugString();
- TString endpoint, name;
- THandleResult handleResult;
- if (!status.IsSuccess()) {
- with_lock (Lock) {
- handleResult = OnErrorImpl({
- status.GetStatus(),
- MakeIssueWithSubIssues("Failed to discover clusters", status.GetIssues())
- });
- }
- ProcessHandleResult(handleResult);
- return;
- }
-
- NYql::TIssues issues;
- EStatus errorStatus = EStatus::INTERNAL_ERROR;
- with_lock (Lock) {
- const Ydb::PersQueue::ClusterDiscovery::WriteSessionClusters& wsClusters = result.write_sessions_clusters(0);
+ TString endpoint, name;
+ THandleResult handleResult;
+ if (!status.IsSuccess()) {
+ with_lock (Lock) {
+ handleResult = OnErrorImpl({
+ status.GetStatus(),
+ MakeIssueWithSubIssues("Failed to discover clusters", status.GetIssues())
+ });
+ }
+ ProcessHandleResult(handleResult);
+ return;
+ }
+
+ NYql::TIssues issues;
+ EStatus errorStatus = EStatus::INTERNAL_ERROR;
+ with_lock (Lock) {
+ const Ydb::PersQueue::ClusterDiscovery::WriteSessionClusters& wsClusters = result.write_sessions_clusters(0);
bool isFirst = true;
- for (const auto& clusterInfo : wsClusters.clusters()) {
- TString normalizedName = clusterInfo.name();
- normalizedName.to_lower();
-
+ for (const auto& clusterInfo : wsClusters.clusters()) {
+ TString normalizedName = clusterInfo.name();
+ normalizedName.to_lower();
+
if(isFirst) {
isFirst = false;
PreferredClusterByCDS = clusterInfo.name();
}
- if (!clusterInfo.available()) {
- if (TargetCluster && TargetCluster == normalizedName) {
- errorStatus = EStatus::UNAVAILABLE;
- issues.AddIssue(TStringBuilder() << "Selected destination cluster: " << normalizedName
- << " is currently disabled");
- break;
- }
- continue;
- }
- if (clusterInfo.endpoint().empty()) {
- issues.AddIssue(TStringBuilder() << "Unexpected reply from cluster discovery. Empty endpoint for cluster "
- << normalizedName);
- } else {
- name = clusterInfo.name();
+ if (!clusterInfo.available()) {
+ if (TargetCluster && TargetCluster == normalizedName) {
+ errorStatus = EStatus::UNAVAILABLE;
+ issues.AddIssue(TStringBuilder() << "Selected destination cluster: " << normalizedName
+ << " is currently disabled");
+ break;
+ }
+ continue;
+ }
+ if (clusterInfo.endpoint().empty()) {
+ issues.AddIssue(TStringBuilder() << "Unexpected reply from cluster discovery. Empty endpoint for cluster "
+ << normalizedName);
+ } else {
+ name = clusterInfo.name();
endpoint = ApplyClusterEndpoint(DbDriverState->DiscoveryEndpoint, clusterInfo.endpoint());
- break;
+ break;
}
}
- if (endpoint.empty()) {
- errorStatus = EStatus::GENERIC_ERROR;
- issues.AddIssue(TStringBuilder() << "Could not get valid endpoint from cluster discovery");
- }
- }
- if (issues) {
- with_lock(Lock) {
- handleResult = OnErrorImpl({errorStatus, std::move(issues)});
- }
- ProcessHandleResult(handleResult);
- return;
- }
- with_lock(Lock) {
- if (!InitialCluster) {
- InitialCluster = name;
- } else if (CurrentCluster != name) { // Switched to another cluster
- Y_VERIFY(CurrentCluster);
- if (name == InitialCluster) { // Returned to initial cluster, disabled SeqNo Shift
- SeqNoShift = 0;
- OnSeqNoShift = false;
- } else { // Switched from initial cluster to second one;
- Y_VERIFY(CurrentCluster == InitialCluster);
- if (!AutoSeqNoMode.Defined() || !(*AutoSeqNoMode))
- OnSeqNoShift = true;
- }
-
- }
- CurrentCluster = name;
- }
+ if (endpoint.empty()) {
+ errorStatus = EStatus::GENERIC_ERROR;
+ issues.AddIssue(TStringBuilder() << "Could not get valid endpoint from cluster discovery");
+ }
+ }
+ if (issues) {
+ with_lock(Lock) {
+ handleResult = OnErrorImpl({errorStatus, std::move(issues)});
+ }
+ ProcessHandleResult(handleResult);
+ return;
+ }
+ with_lock(Lock) {
+ if (!InitialCluster) {
+ InitialCluster = name;
+ } else if (CurrentCluster != name) { // Switched to another cluster
+ Y_VERIFY(CurrentCluster);
+ if (name == InitialCluster) { // Returned to initial cluster, disabled SeqNo Shift
+ SeqNoShift = 0;
+ OnSeqNoShift = false;
+ } else { // Switched from initial cluster to second one;
+ Y_VERIFY(CurrentCluster == InitialCluster);
+ if (!AutoSeqNoMode.Defined() || !(*AutoSeqNoMode))
+ OnSeqNoShift = true;
+ }
+
+ }
+ CurrentCluster = name;
+ }
DoConnect(TDuration::Zero(), endpoint);
-
-}
-
-void TWriteSession::InitWriter() { // No Lock, very initial start - no race yet as well.
- CompressionExecutor = Settings.CompressionExecutor_;
- IExecutor::TPtr executor;
- executor = new TSyncExecutor();
- executor->Start();
- Executor = std::move(executor);
-
- Settings.CompressionExecutor_->Start();
- Settings.EventHandlers_.HandlersExecutor_->Start();
-
-}
-// Client method
-NThreading::TFuture<ui64> TWriteSession::GetInitSeqNo() {
+
+}
+
+void TWriteSession::InitWriter() { // No Lock, very initial start - no race yet as well.
+ CompressionExecutor = Settings.CompressionExecutor_;
+ IExecutor::TPtr executor;
+ executor = new TSyncExecutor();
+ executor->Start();
+ Executor = std::move(executor);
+
+ Settings.CompressionExecutor_->Start();
+ Settings.EventHandlers_.HandlersExecutor_->Start();
+
+}
+// Client method
+NThreading::TFuture<ui64> TWriteSession::GetInitSeqNo() {
if (Settings.ValidateSeqNo_) {
- if (AutoSeqNoMode.Defined() && *AutoSeqNoMode) {
+ if (AutoSeqNoMode.Defined() && *AutoSeqNoMode) {
DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call GetInitSeqNo in Auto SeqNo mode";
ThrowFatalError("Cannot call GetInitSeqNo in Auto SeqNo mode");
- }
+ }
else
AutoSeqNoMode = false;
}
- return InitSeqNoPromise.GetFuture();
-}
-
+ return InitSeqNoPromise.GetFuture();
+}
+
TString DebugString(const TWriteSessionEvent::TEvent& event) {
return std::visit([](const auto& ev) { return ev.DebugString(); }, event);
}
-// Client method
-TMaybe<TWriteSessionEvent::TEvent> TWriteSession::GetEvent(bool block) {
- return EventsQueue->GetEvent(block);
-}
-
-// Client method
-TVector<TWriteSessionEvent::TEvent> TWriteSession::GetEvents(bool block, TMaybe<size_t> maxEventsCount) {
+// Client method
+TMaybe<TWriteSessionEvent::TEvent> TWriteSession::GetEvent(bool block) {
+ return EventsQueue->GetEvent(block);
+}
+
+// Client method
+TVector<TWriteSessionEvent::TEvent> TWriteSession::GetEvents(bool block, TMaybe<size_t> maxEventsCount) {
return EventsQueue->GetEvents(block, maxEventsCount);
-}
-
-// Only called under lock
-ui64 TWriteSession::GetNextSeqNoImpl(const TMaybe<ui64>& seqNo) {
- ui64 seqNoValue = LastSeqNo + 1;
- if (!AutoSeqNoMode.Defined()) {
- AutoSeqNoMode = !seqNo.Defined();
- //! Disable SeqNo shift for manual SeqNo mode;
- if (seqNo.Defined()) {
- OnSeqNoShift = false;
- SeqNoShift = 0;
- }
+}
+
+// Only called under lock
+ui64 TWriteSession::GetNextSeqNoImpl(const TMaybe<ui64>& seqNo) {
+ ui64 seqNoValue = LastSeqNo + 1;
+ if (!AutoSeqNoMode.Defined()) {
+ AutoSeqNoMode = !seqNo.Defined();
+ //! Disable SeqNo shift for manual SeqNo mode;
+ if (seqNo.Defined()) {
+ OnSeqNoShift = false;
+ SeqNoShift = 0;
+ }
}
if (seqNo.Defined()) {
if (*AutoSeqNoMode) {
DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode";
ThrowFatalError(
- "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode"
- );
- } else {
- seqNoValue = *seqNo;
- }
+ "Cannot call write() with defined SeqNo on WriteSession running in auto-seqNo mode"
+ );
+ } else {
+ seqNoValue = *seqNo;
+ }
} else if (!(*AutoSeqNoMode)) {
DbDriverState->Log << TLOG_ERR << LogPrefix() << "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode";
ThrowFatalError(
- "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode"
- );
- }
- LastSeqNo = seqNoValue;
- return seqNoValue;
-}
-inline void TWriteSession::CheckHandleResultImpl(THandleResult& result) {
- result.DoSetSeqNo = result.DoStop && !InitSeqNoSetDone && (InitSeqNoSetDone = true);
-}
-
-void TWriteSession::ProcessHandleResult(THandleResult& result) {
- if (result.DoRestart) {
- Start(result.StartDelay);
- } else if (result.DoSetSeqNo) {
- InitSeqNoPromise.SetException("session closed");
- }
-}
-
-NThreading::TFuture<void> TWriteSession::WaitEvent() {
- return EventsQueue->WaitEvent();
-}
-
-// Client method.
+ "Cannot call write() without defined SeqNo on WriteSession running in manual-seqNo mode"
+ );
+ }
+ LastSeqNo = seqNoValue;
+ return seqNoValue;
+}
+inline void TWriteSession::CheckHandleResultImpl(THandleResult& result) {
+ result.DoSetSeqNo = result.DoStop && !InitSeqNoSetDone && (InitSeqNoSetDone = true);
+}
+
+void TWriteSession::ProcessHandleResult(THandleResult& result) {
+ if (result.DoRestart) {
+ Start(result.StartDelay);
+ } else if (result.DoSetSeqNo) {
+ InitSeqNoPromise.SetException("session closed");
+ }
+}
+
+NThreading::TFuture<void> TWriteSession::WaitEvent() {
+ return EventsQueue->WaitEvent();
+}
+
+// Client method.
void TWriteSession::WriteInternal(
TContinuationToken&&, TStringBuf data, TMaybe<ECodec> codec, ui32 originalSize, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp
) {
- TInstant createdAtValue = createTimestamp.Defined() ? *createTimestamp : TInstant::Now();
- bool readyToAccept = false;
- size_t bufferSize = data.size();
- with_lock(Lock) {
+ TInstant createdAtValue = createTimestamp.Defined() ? *createTimestamp : TInstant::Now();
+ bool readyToAccept = false;
+ size_t bufferSize = data.size();
+ with_lock(Lock) {
CurrentBatch.Add(GetNextSeqNoImpl(seqNo), createdAtValue, data, codec, originalSize);
-
- FlushWriteIfRequiredImpl();
- readyToAccept = OnMemoryUsageChangedImpl(bufferSize).NowOk;
- }
- if (readyToAccept) {
- EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
- }
-}
-
+
+ FlushWriteIfRequiredImpl();
+ readyToAccept = OnMemoryUsageChangedImpl(bufferSize).NowOk;
+ }
+ if (readyToAccept) {
+ EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
+ }
+}
+
// Client method.
void TWriteSession::WriteEncoded(
TContinuationToken&& token, TStringBuf data, ECodec codec, ui32 originalSize, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp
@@ -366,275 +366,275 @@ void TWriteSession::Write(
}
-// Only called under lock.
-TWriteSession::THandleResult TWriteSession::OnErrorImpl(NYdb::TPlainStatus&& status) {
- (*Counters->Errors)++;
- auto result = RestartImpl(status);
- if (result.DoStop) {
- CloseImpl(status.Status, std::move(status.Issues));
- }
- return result;
-}
-
-// No lock
+// Only called under lock.
+TWriteSession::THandleResult TWriteSession::OnErrorImpl(NYdb::TPlainStatus&& status) {
+ (*Counters->Errors)++;
+ auto result = RestartImpl(status);
+ if (result.DoStop) {
+ CloseImpl(status.Status, std::move(status.Issues));
+ }
+ return result;
+}
+
+// No lock
void TWriteSession::DoConnect(const TDuration& delay, const TString& endpoint) {
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Start write session. Will connect to endpoint: " << endpoint;
-
- NGrpc::IQueueClientContextPtr prevConnectContext;
- NGrpc::IQueueClientContextPtr prevConnectTimeoutContext;
- NGrpc::IQueueClientContextPtr prevConnectDelayContext;
- NGrpc::IQueueClientContextPtr connectContext = nullptr;
+
+ NGrpc::IQueueClientContextPtr prevConnectContext;
+ NGrpc::IQueueClientContextPtr prevConnectTimeoutContext;
+ NGrpc::IQueueClientContextPtr prevConnectDelayContext;
+ NGrpc::IQueueClientContextPtr connectContext = nullptr;
NGrpc::IQueueClientContextPtr connectDelayContext = nullptr;
- NGrpc::IQueueClientContextPtr connectTimeoutContext = nullptr;
- TRpcRequestSettings reqSettings;
- std::shared_ptr<IWriteSessionConnectionProcessorFactory> connectionFactory;
- with_lock(Lock) {
- ++ConnectionGeneration;
- auto subclient = Client->GetClientForEndpoint(endpoint);
- connectionFactory = subclient->CreateWriteSessionConnectionProcessorFactory();
- auto clientContext = subclient->CreateContext();
- ConnectionFactory = connectionFactory;
-
- ClientContext = std::move(clientContext);
- ServerMessage = std::make_shared<TServerMessage>();
-
+ NGrpc::IQueueClientContextPtr connectTimeoutContext = nullptr;
+ TRpcRequestSettings reqSettings;
+ std::shared_ptr<IWriteSessionConnectionProcessorFactory> connectionFactory;
+ with_lock(Lock) {
+ ++ConnectionGeneration;
+ auto subclient = Client->GetClientForEndpoint(endpoint);
+ connectionFactory = subclient->CreateWriteSessionConnectionProcessorFactory();
+ auto clientContext = subclient->CreateContext();
+ ConnectionFactory = connectionFactory;
+
+ ClientContext = std::move(clientContext);
+ ServerMessage = std::make_shared<TServerMessage>();
+
if (!ClientContext) {
AbortImpl();
// Grpc and WriteSession is closing right now.
return;
}
- connectContext = ClientContext->CreateContext();
+ connectContext = ClientContext->CreateContext();
if (delay)
connectDelayContext = ClientContext->CreateContext();
- connectTimeoutContext = ClientContext->CreateContext();
-
- // Previous operations contexts.
-
- // Set new context
- prevConnectContext = std::exchange(ConnectContext, connectContext);
- prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext);
+ connectTimeoutContext = ClientContext->CreateContext();
+
+ // Previous operations contexts.
+
+ // Set new context
+ prevConnectContext = std::exchange(ConnectContext, connectContext);
+ prevConnectTimeoutContext = std::exchange(ConnectTimeoutContext, connectTimeoutContext);
prevConnectDelayContext = std::exchange(ConnectDelayContext, connectDelayContext);
- Y_ASSERT(ConnectContext);
- Y_ASSERT(ConnectTimeoutContext);
-
- // Cancel previous operations.
- Cancel(prevConnectContext);
+ Y_ASSERT(ConnectContext);
+ Y_ASSERT(ConnectTimeoutContext);
+
+ // Cancel previous operations.
+ Cancel(prevConnectContext);
if (prevConnectDelayContext)
Cancel(prevConnectDelayContext);
- Cancel(prevConnectTimeoutContext);
- Y_ASSERT(connectContext);
- Y_ASSERT(connectTimeoutContext);
- reqSettings = TRpcRequestSettings::Make(Settings);
- }
- auto connectCallback = [weakThis = weak_from_this(), connectContext = connectContext]
- (TPlainStatus&& st, typename IProcessor::TPtr&& processor) {
- if (auto sharedThis = weakThis.lock()) {
- sharedThis->OnConnect(std::move(st), std::move(processor), connectContext);
- }
- };
-
- auto connectTimeoutCallback = [weakThis = weak_from_this(), connectTimeoutContext = connectTimeoutContext]
- (bool ok) {
- if (ok) {
- if (auto sharedThis = weakThis.lock()) {
- sharedThis->OnConnectTimeout(connectTimeoutContext);
- }
- }
- };
-
- connectionFactory->CreateProcessor(
- std::move(connectCallback),
- reqSettings,
- std::move(connectContext),
- TDuration::Seconds(30) /* connect timeout */, // TODO: make connect timeout setting.
- std::move(connectTimeoutContext),
+ Cancel(prevConnectTimeoutContext);
+ Y_ASSERT(connectContext);
+ Y_ASSERT(connectTimeoutContext);
+ reqSettings = TRpcRequestSettings::Make(Settings);
+ }
+ auto connectCallback = [weakThis = weak_from_this(), connectContext = connectContext]
+ (TPlainStatus&& st, typename IProcessor::TPtr&& processor) {
+ if (auto sharedThis = weakThis.lock()) {
+ sharedThis->OnConnect(std::move(st), std::move(processor), connectContext);
+ }
+ };
+
+ auto connectTimeoutCallback = [weakThis = weak_from_this(), connectTimeoutContext = connectTimeoutContext]
+ (bool ok) {
+ if (ok) {
+ if (auto sharedThis = weakThis.lock()) {
+ sharedThis->OnConnectTimeout(connectTimeoutContext);
+ }
+ }
+ };
+
+ connectionFactory->CreateProcessor(
+ std::move(connectCallback),
+ reqSettings,
+ std::move(connectContext),
+ TDuration::Seconds(30) /* connect timeout */, // TODO: make connect timeout setting.
+ std::move(connectTimeoutContext),
std::move(connectTimeoutCallback),
delay,
std::move(connectDelayContext)
- );
-}
-
-// RPC callback.
-void TWriteSession::OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext) {
+ );
+}
+
+// RPC callback.
+void TWriteSession::OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext) {
DbDriverState->Log << TLOG_ERR << LogPrefix() << "Write session: connect timeout";
- THandleResult handleResult;
- with_lock (Lock) {
- if (ConnectTimeoutContext == connectTimeoutContext) {
- Cancel(ConnectContext);
- ConnectContext = nullptr;
- ConnectTimeoutContext = nullptr;
- ConnectDelayContext = nullptr;
- } else {
- return;
- }
+ THandleResult handleResult;
+ with_lock (Lock) {
+ if (ConnectTimeoutContext == connectTimeoutContext) {
+ Cancel(ConnectContext);
+ ConnectContext = nullptr;
+ ConnectTimeoutContext = nullptr;
+ ConnectDelayContext = nullptr;
+ } else {
+ return;
+ }
TStringBuilder description;
description << "Failed to establish connection to server. Attempts done: " << ConnectionAttemptsDone;
- handleResult = RestartImpl(TPlainStatus(EStatus::TIMEOUT, description));
- if (handleResult.DoStop) {
- CloseImpl(
- EStatus::TIMEOUT,
+ handleResult = RestartImpl(TPlainStatus(EStatus::TIMEOUT, description));
+ if (handleResult.DoStop) {
+ CloseImpl(
+ EStatus::TIMEOUT,
description
- );
- }
- }
- ProcessHandleResult(handleResult);
-}
-
-// RPC callback.
-void TWriteSession::OnConnect(
- TPlainStatus&& st, typename IProcessor::TPtr&& processor, const NGrpc::IQueueClientContextPtr& connectContext
-) {
- THandleResult handleResult;
- with_lock (Lock) {
- if (ConnectContext == connectContext) {
- Cancel(ConnectTimeoutContext);
- ConnectContext = nullptr;
- ConnectTimeoutContext = nullptr;
- ConnectDelayContext = nullptr;
-
- if (st.Ok()) {
- Processor = std::move(processor);
- InitImpl();
- // Still should call ReadFromProcessor();
- }
- } else {
- return;
- }
- if (!st.Ok()) {
- handleResult = RestartImpl(st);
- if (handleResult.DoStop) {
- CloseImpl(
- st.Status,
- MakeIssueWithSubIssues(
- TStringBuilder() << "Failed to establish connection to server \"" << st.Endpoint
- << "\". Attempts done: " << ConnectionAttemptsDone,
- st.Issues
- )
- );
- }
- }
- }
- if (st.Ok())
- ReadFromProcessor(); // Out of Init
- ProcessHandleResult(handleResult);
-}
-
-// Produce init request for session.
-// Only called under lock.
-void TWriteSession::InitImpl() {
- Ydb::PersQueue::V1::StreamingWriteClientMessage req;
- auto* init = req.mutable_init_request();
- init->set_topic(Settings.Path_);
- init->set_message_group_id(Settings.MessageGroupId_);
- if (Settings.PartitionGroupId_) {
- init->set_partition_group_id(*Settings.PartitionGroupId_);
- }
- init->set_max_supported_block_format_version(0);
+ );
+ }
+ }
+ ProcessHandleResult(handleResult);
+}
+
+// RPC callback.
+void TWriteSession::OnConnect(
+ TPlainStatus&& st, typename IProcessor::TPtr&& processor, const NGrpc::IQueueClientContextPtr& connectContext
+) {
+ THandleResult handleResult;
+ with_lock (Lock) {
+ if (ConnectContext == connectContext) {
+ Cancel(ConnectTimeoutContext);
+ ConnectContext = nullptr;
+ ConnectTimeoutContext = nullptr;
+ ConnectDelayContext = nullptr;
+
+ if (st.Ok()) {
+ Processor = std::move(processor);
+ InitImpl();
+ // Still should call ReadFromProcessor();
+ }
+ } else {
+ return;
+ }
+ if (!st.Ok()) {
+ handleResult = RestartImpl(st);
+ if (handleResult.DoStop) {
+ CloseImpl(
+ st.Status,
+ MakeIssueWithSubIssues(
+ TStringBuilder() << "Failed to establish connection to server \"" << st.Endpoint
+ << "\". Attempts done: " << ConnectionAttemptsDone,
+ st.Issues
+ )
+ );
+ }
+ }
+ }
+ if (st.Ok())
+ ReadFromProcessor(); // Out of Init
+ ProcessHandleResult(handleResult);
+}
+
+// Produce init request for session.
+// Only called under lock.
+void TWriteSession::InitImpl() {
+ Ydb::PersQueue::V1::StreamingWriteClientMessage req;
+ auto* init = req.mutable_init_request();
+ init->set_topic(Settings.Path_);
+ init->set_message_group_id(Settings.MessageGroupId_);
+ if (Settings.PartitionGroupId_) {
+ init->set_partition_group_id(*Settings.PartitionGroupId_);
+ }
+ init->set_max_supported_block_format_version(0);
init->set_preferred_cluster(PreferredClusterByCDS);
-
- for (const auto& attr : Settings.Meta_.Fields) {
- (*init->mutable_session_meta())[attr.first] = attr.second;
- }
+
+ for (const auto& attr : Settings.Meta_.Fields) {
+ (*init->mutable_session_meta())[attr.first] = attr.second;
+ }
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: send init request: "<< req.ShortDebugString();
- WriteToProcessorImpl(std::move(req));
-}
-
-// Called under lock. Invokes Processor->Write, which is assumed to be deadlock-safe
-void TWriteSession::WriteToProcessorImpl(TWriteSession::TClientMessage&& req) {
- Y_ASSERT(Processor);
- if (Aborting)
- return;
- auto callback = [weakThis = weak_from_this(), connectionGeneration = ConnectionGeneration](NGrpc::TGrpcStatus&& grpcStatus) {
- if (auto sharedThis = weakThis.lock()) {
- sharedThis->OnWriteDone(std::move(grpcStatus), connectionGeneration);
- }
- };
-
- Processor->Write(std::move(req), callback);
-}
-
-void TWriteSession::ReadFromProcessor() {
- Y_ASSERT(Processor);
- IProcessor::TPtr prc;
- ui64 generation;
- with_lock(Lock) {
- prc = Processor;
- generation = ConnectionGeneration;
- }
- auto callback = [weakThis = weak_from_this(), connectionGeneration = generation, processor = prc, serverMessage = ServerMessage]
- (NGrpc::TGrpcStatus&& grpcStatus) {
- if (auto sharedThis = weakThis.lock()) {
- sharedThis->OnReadDone(std::move(grpcStatus), connectionGeneration);
- }
- };
- prc->Read(ServerMessage.get(), std::move(callback));
-}
-
-void TWriteSession::OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration) {
- THandleResult handleResult;
- with_lock (Lock) {
- if (connectionGeneration != ConnectionGeneration) {
- return; // Message from previous connection. Ignore.
- }
- if (Aborting) {
- return;
- }
- if(!status.Ok()) {
- handleResult = OnErrorImpl(status);
- }
- }
- ProcessHandleResult(handleResult);
-}
-
-void TWriteSession::OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration) {
- TPlainStatus errorStatus;
- TProcessSrvMessageResult processResult;
- bool needSetValue = false;
- if (!grpcStatus.Ok()) {
- errorStatus = TPlainStatus(std::move(grpcStatus));
- }
- bool doRead = false;
- with_lock (Lock) {
- UpdateTimedCountersImpl();
- if (connectionGeneration != ConnectionGeneration) {
- return; // Message from previous connection. Ignore.
- }
- if (errorStatus.Ok()) {
- if (IsErrorMessage(*ServerMessage)) {
- errorStatus = MakeErrorFromProto(*ServerMessage);
- } else {
- processResult = ProcessServerMessageImpl();
- needSetValue = !InitSeqNoSetDone && processResult.InitSeqNo.Defined() && (InitSeqNoSetDone = true);
- if (errorStatus.Ok() && processResult.Ok) {
- doRead = true;
- }
- }
- }
- }
- if (doRead)
- ReadFromProcessor();
-
- with_lock(Lock) {
- if (!errorStatus.Ok()) {
- if (processResult.Ok) { // Otherwise, OnError was already called
- processResult.HandleResult = RestartImpl(errorStatus);
- }
- }
- if (processResult.HandleResult.DoStop) {
- CloseImpl(std::move(errorStatus));
- }
- }
- for (auto& event : processResult.Events) {
- EventsQueue->PushEvent(std::move(event));
- }
- if (needSetValue) {
- InitSeqNoPromise.SetValue(*processResult.InitSeqNo);
- processResult.HandleResult.DoSetSeqNo = false; // Redundant. Just in case.
- }
- ProcessHandleResult(processResult.HandleResult);
-}
-
+ WriteToProcessorImpl(std::move(req));
+}
+
+// Called under lock. Invokes Processor->Write, which is assumed to be deadlock-safe
+void TWriteSession::WriteToProcessorImpl(TWriteSession::TClientMessage&& req) {
+ Y_ASSERT(Processor);
+ if (Aborting)
+ return;
+ auto callback = [weakThis = weak_from_this(), connectionGeneration = ConnectionGeneration](NGrpc::TGrpcStatus&& grpcStatus) {
+ if (auto sharedThis = weakThis.lock()) {
+ sharedThis->OnWriteDone(std::move(grpcStatus), connectionGeneration);
+ }
+ };
+
+ Processor->Write(std::move(req), callback);
+}
+
+void TWriteSession::ReadFromProcessor() {
+ Y_ASSERT(Processor);
+ IProcessor::TPtr prc;
+ ui64 generation;
+ with_lock(Lock) {
+ prc = Processor;
+ generation = ConnectionGeneration;
+ }
+ auto callback = [weakThis = weak_from_this(), connectionGeneration = generation, processor = prc, serverMessage = ServerMessage]
+ (NGrpc::TGrpcStatus&& grpcStatus) {
+ if (auto sharedThis = weakThis.lock()) {
+ sharedThis->OnReadDone(std::move(grpcStatus), connectionGeneration);
+ }
+ };
+ prc->Read(ServerMessage.get(), std::move(callback));
+}
+
+void TWriteSession::OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration) {
+ THandleResult handleResult;
+ with_lock (Lock) {
+ if (connectionGeneration != ConnectionGeneration) {
+ return; // Message from previous connection. Ignore.
+ }
+ if (Aborting) {
+ return;
+ }
+ if(!status.Ok()) {
+ handleResult = OnErrorImpl(status);
+ }
+ }
+ ProcessHandleResult(handleResult);
+}
+
+void TWriteSession::OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration) {
+ TPlainStatus errorStatus;
+ TProcessSrvMessageResult processResult;
+ bool needSetValue = false;
+ if (!grpcStatus.Ok()) {
+ errorStatus = TPlainStatus(std::move(grpcStatus));
+ }
+ bool doRead = false;
+ with_lock (Lock) {
+ UpdateTimedCountersImpl();
+ if (connectionGeneration != ConnectionGeneration) {
+ return; // Message from previous connection. Ignore.
+ }
+ if (errorStatus.Ok()) {
+ if (IsErrorMessage(*ServerMessage)) {
+ errorStatus = MakeErrorFromProto(*ServerMessage);
+ } else {
+ processResult = ProcessServerMessageImpl();
+ needSetValue = !InitSeqNoSetDone && processResult.InitSeqNo.Defined() && (InitSeqNoSetDone = true);
+ if (errorStatus.Ok() && processResult.Ok) {
+ doRead = true;
+ }
+ }
+ }
+ }
+ if (doRead)
+ ReadFromProcessor();
+
+ with_lock(Lock) {
+ if (!errorStatus.Ok()) {
+ if (processResult.Ok) { // Otherwise, OnError was already called
+ processResult.HandleResult = RestartImpl(errorStatus);
+ }
+ }
+ if (processResult.HandleResult.DoStop) {
+ CloseImpl(std::move(errorStatus));
+ }
+ }
+ for (auto& event : processResult.Events) {
+ EventsQueue->PushEvent(std::move(event));
+ }
+ if (needSetValue) {
+ InitSeqNoPromise.SetValue(*processResult.InitSeqNo);
+ processResult.HandleResult.DoSetSeqNo = false; // Redundant. Just in case.
+ }
+ ProcessHandleResult(processResult.HandleResult);
+}
+
TString TWriteSession::LogPrefix() const {
return TStringBuilder() << "MessageGroupId [" << Settings.MessageGroupId_ << "] SessionId [" << SessionId << "] ";
}
@@ -662,46 +662,46 @@ TString TWriteSessionEvent::TReadyToAcceptEvent::DebugString() const {
}
-TWriteSession::TProcessSrvMessageResult TWriteSession::ProcessServerMessageImpl() {
- TProcessSrvMessageResult result;
- switch (ServerMessage->GetServerMessageCase()) {
- case TServerMessage::SERVER_MESSAGE_NOT_SET: {
- SessionEstablished = false;
- result.HandleResult = OnErrorImpl({
- static_cast<NYdb::EStatus>(ServerMessage->status()),
- {NYql::TIssue{ServerMessage->DebugString()}}
- });
- result.Ok = false;
- break;
- }
- case TServerMessage::kInitResponse: {
- const auto& initResponse = ServerMessage->init_response();
+TWriteSession::TProcessSrvMessageResult TWriteSession::ProcessServerMessageImpl() {
+ TProcessSrvMessageResult result;
+ switch (ServerMessage->GetServerMessageCase()) {
+ case TServerMessage::SERVER_MESSAGE_NOT_SET: {
+ SessionEstablished = false;
+ result.HandleResult = OnErrorImpl({
+ static_cast<NYdb::EStatus>(ServerMessage->status()),
+ {NYql::TIssue{ServerMessage->DebugString()}}
+ });
+ result.Ok = false;
+ break;
+ }
+ case TServerMessage::kInitResponse: {
+ const auto& initResponse = ServerMessage->init_response();
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session established. Init response: " << initResponse.ShortDebugString();
- SessionId = initResponse.session_id();
- PartitionId = initResponse.partition_id();
- ui64 newLastSeqNo = initResponse.last_sequence_number();
- // SeqNo increased, so there's a risk of loss, apply SeqNo shift.
- // MinUnsentSeqNo must be > 0 if anything was ever sent yet
- if (MinUnsentSeqNo && OnSeqNoShift && newLastSeqNo > MinUnsentSeqNo) {
- SeqNoShift = newLastSeqNo - MinUnsentSeqNo;
- }
- result.InitSeqNo = newLastSeqNo;
- LastSeqNo = newLastSeqNo;
-
- SessionEstablished = true;
- LastCountersUpdateTs = TInstant::Now();
- SessionStartedTs = TInstant::Now();
- OnErrorResolved();
-
- //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
- result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
- // Kickstart send after session reestablishment
- SendImpl();
- break;
- }
- case TServerMessage::kBatchWriteResponse: {
- TWriteSessionEvent::TAcksEvent acksEvent;
- const auto& batchWriteResponse = ServerMessage->batch_write_response();
+ SessionId = initResponse.session_id();
+ PartitionId = initResponse.partition_id();
+ ui64 newLastSeqNo = initResponse.last_sequence_number();
+ // SeqNo increased, so there's a risk of loss, apply SeqNo shift.
+ // MinUnsentSeqNo must be > 0 if anything was ever sent yet
+ if (MinUnsentSeqNo && OnSeqNoShift && newLastSeqNo > MinUnsentSeqNo) {
+ SeqNoShift = newLastSeqNo - MinUnsentSeqNo;
+ }
+ result.InitSeqNo = newLastSeqNo;
+ LastSeqNo = newLastSeqNo;
+
+ SessionEstablished = true;
+ LastCountersUpdateTs = TInstant::Now();
+ SessionStartedTs = TInstant::Now();
+ OnErrorResolved();
+
+ //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
+ result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
+ // Kickstart send after session reestablishment
+ SendImpl();
+ break;
+ }
+ case TServerMessage::kBatchWriteResponse: {
+ TWriteSessionEvent::TAcksEvent acksEvent;
+ const auto& batchWriteResponse = ServerMessage->batch_write_response();
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session got write response: " << batchWriteResponse.ShortDebugString();
TWriteStat::TPtr writeStat = new TWriteStat{};
const auto& stat = batchWriteResponse.write_statistics();
@@ -710,285 +710,285 @@ TWriteSession::TProcessSrvMessageResult TWriteSession::ProcessServerMessageImpl(
writeStat->PartitionQuotedTime = TDuration::MilliSeconds(stat.throttled_on_partition_duration_ms());
writeStat->TopicQuotedTime = TDuration::MilliSeconds(stat.throttled_on_topic_duration_ms());
- for (size_t messageIndex = 0, endIndex = batchWriteResponse.sequence_numbers_size(); messageIndex != endIndex; ++messageIndex) {
- // TODO: Fill writer statistics
- ui64 sequenceNumber = batchWriteResponse.sequence_numbers(messageIndex);
-
- acksEvent.Acks.push_back(TWriteSessionEvent::TWriteAck{
- sequenceNumber - SeqNoShift,
- batchWriteResponse.already_written(messageIndex) ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN:
- TWriteSessionEvent::TWriteAck::EES_WRITTEN,
- TWriteSessionEvent::TWriteAck::TWrittenMessageDetails {
- static_cast<ui64>(batchWriteResponse.offsets(messageIndex)),
- PartitionId,
- },
+ for (size_t messageIndex = 0, endIndex = batchWriteResponse.sequence_numbers_size(); messageIndex != endIndex; ++messageIndex) {
+ // TODO: Fill writer statistics
+ ui64 sequenceNumber = batchWriteResponse.sequence_numbers(messageIndex);
+
+ acksEvent.Acks.push_back(TWriteSessionEvent::TWriteAck{
+ sequenceNumber - SeqNoShift,
+ batchWriteResponse.already_written(messageIndex) ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN:
+ TWriteSessionEvent::TWriteAck::EES_WRITTEN,
+ TWriteSessionEvent::TWriteAck::TWrittenMessageDetails {
+ static_cast<ui64>(batchWriteResponse.offsets(messageIndex)),
+ PartitionId,
+ },
writeStat,
- });
-
- if (CleanupOnAcknowledged(sequenceNumber - SeqNoShift)) {
- result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
- }
- }
- //EventsQueue->PushEvent(std::move(acksEvent));
- result.Events.emplace_back(std::move(acksEvent));
- break;
- }
- case TServerMessage::kUpdateTokenResponse: {
+ });
+
+ if (CleanupOnAcknowledged(sequenceNumber - SeqNoShift)) {
+ result.Events.emplace_back(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
+ }
+ }
+ //EventsQueue->PushEvent(std::move(acksEvent));
+ result.Events.emplace_back(std::move(acksEvent));
+ break;
+ }
+ case TServerMessage::kUpdateTokenResponse: {
UpdateTokenInProgress = false;
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: token updated successfully";
UpdateTokenIfNeededImpl();
- break;
- }
- }
- return result;
-}
-
-bool TWriteSession::CleanupOnAcknowledged(ui64 sequenceNumber) {
- bool result = false;
+ break;
+ }
+ }
+ return result;
+}
+
+bool TWriteSession::CleanupOnAcknowledged(ui64 sequenceNumber) {
+ bool result = false;
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: acknoledged message " << sequenceNumber;
- UpdateTimedCountersImpl();
- const auto& sentFront = SentOriginalMessages.front();
- ui64 size = 0;
- ui64 compressedSize = 0;
- if(!SentPackedMessage.empty() && SentPackedMessage.front().Offset == sequenceNumber) {
- auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size());
- result = memoryUsage.NowOk && !memoryUsage.WasOk;
- //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
+ UpdateTimedCountersImpl();
+ const auto& sentFront = SentOriginalMessages.front();
+ ui64 size = 0;
+ ui64 compressedSize = 0;
+ if(!SentPackedMessage.empty() && SentPackedMessage.front().Offset == sequenceNumber) {
+ auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size());
+ result = memoryUsage.NowOk && !memoryUsage.WasOk;
+ //EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
const auto& front = SentPackedMessage.front();
- if (front.Compressed) {
- compressedSize = front.Data.size();
- } else {
- size = front.Data.size();
- }
-
- (*Counters->MessagesWritten) += front.MessageCount;
- (*Counters->MessagesInflight) -= front.MessageCount;
+ if (front.Compressed) {
+ compressedSize = front.Data.size();
+ } else {
+ size = front.Data.size();
+ }
+
+ (*Counters->MessagesWritten) += front.MessageCount;
+ (*Counters->MessagesInflight) -= front.MessageCount;
(*Counters->BytesWritten) += front.OriginalSize;
-
- SentPackedMessage.pop();
- } else {
- size = sentFront.Size;
- (*Counters->BytesWritten) += sentFront.Size;
- (*Counters->MessagesWritten)++;
- (*Counters->MessagesInflight)--;
- }
-
- (*Counters->BytesInflightCompressed) -= compressedSize;
- (*Counters->BytesWrittenCompressed) += compressedSize;
- (*Counters->BytesInflightUncompressed) -= size;
-
- Y_VERIFY(Counters->BytesInflightCompressed->Val() >= 0);
- Y_VERIFY(Counters->BytesInflightUncompressed->Val() >= 0);
-
- Y_VERIFY(sentFront.SeqNo == sequenceNumber);
-
- (*Counters->BytesInflightTotal) = MemoryUsage;
- SentOriginalMessages.pop();
- return result;
-}
-
-// Only called under Lock
-TMemoryUsageChange TWriteSession::OnMemoryUsageChangedImpl(i64 diff) {
- bool wasOk = MemoryUsage <= Settings.MaxMemoryUsage_;
+
+ SentPackedMessage.pop();
+ } else {
+ size = sentFront.Size;
+ (*Counters->BytesWritten) += sentFront.Size;
+ (*Counters->MessagesWritten)++;
+ (*Counters->MessagesInflight)--;
+ }
+
+ (*Counters->BytesInflightCompressed) -= compressedSize;
+ (*Counters->BytesWrittenCompressed) += compressedSize;
+ (*Counters->BytesInflightUncompressed) -= size;
+
+ Y_VERIFY(Counters->BytesInflightCompressed->Val() >= 0);
+ Y_VERIFY(Counters->BytesInflightUncompressed->Val() >= 0);
+
+ Y_VERIFY(sentFront.SeqNo == sequenceNumber);
+
+ (*Counters->BytesInflightTotal) = MemoryUsage;
+ SentOriginalMessages.pop();
+ return result;
+}
+
+// Only called under Lock
+TMemoryUsageChange TWriteSession::OnMemoryUsageChangedImpl(i64 diff) {
+ bool wasOk = MemoryUsage <= Settings.MaxMemoryUsage_;
//if (diff < 0) {
// Y_VERIFY(MemoryUsage >= static_cast<size_t>(std::abs(diff)));
//}
- MemoryUsage += diff;
- bool nowOk = MemoryUsage <= Settings.MaxMemoryUsage_;
- if (wasOk != nowOk) {
- if (wasOk) {
+ MemoryUsage += diff;
+ bool nowOk = MemoryUsage <= Settings.MaxMemoryUsage_;
+ if (wasOk != nowOk) {
+ if (wasOk) {
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Estimated memory usage " << MemoryUsage << "[B] reached maximum (" << Settings.MaxMemoryUsage_ << "[B])";
- }
- else {
+ }
+ else {
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Estimated memory usage got back to normal " << MemoryUsage << "[B]";
- }
- }
- return {wasOk, nowOk};
-}
-
+ }
+ }
+ return {wasOk, nowOk};
+}
+
TBuffer CompressBuffer(TVector<TStringBuf>& data, ECodec codec, i32 level) {
- TBuffer result;
+ TBuffer result;
THolder<IOutputStream> coder = NCompressionDetails::CreateCoder(codec, result, level);
- for (auto& buffer : data) {
- coder->Write(buffer.data(), buffer.size());
- }
- coder->Finish();
- return result;
-}
-
-// May call OnCompressed with sync executor. No external lock.
+ for (auto& buffer : data) {
+ coder->Write(buffer.data(), buffer.size());
+ }
+ coder->Finish();
+ return result;
+}
+
+// May call OnCompressed with sync executor. No external lock.
void TWriteSession::CompressImpl(TBlock&& block_) {
- auto weakThis = weak_from_this();
+ auto weakThis = weak_from_this();
bool isSyncCompression = !CompressionExecutor->IsAsync();
Y_VERIFY(block_.Valid);
-
+
std::shared_ptr<TBlock> blockPtr(std::make_shared<TBlock>());
blockPtr->Move(block_);
auto lambda = [weakThis, codec = Settings.Codec_, level = Settings.CompressionLevel_,
isSyncCompression, blockPtr]() mutable
- {
- if (auto sharedThis = weakThis.lock()) {
+ {
+ if (auto sharedThis = weakThis.lock()) {
Y_VERIFY(!blockPtr->Compressed);
- auto compressedData = CompressBuffer(
+ auto compressedData = CompressBuffer(
blockPtr->OriginalDataRefs, codec, level
- );
- Y_VERIFY(!compressedData.Empty());
+ );
+ Y_VERIFY(!compressedData.Empty());
blockPtr->Data = std::move(compressedData);
blockPtr->Compressed = true;
blockPtr->CodecID = GetCodecId(sharedThis->Settings.Codec_);
sharedThis->OnCompressed(std::move(*blockPtr), isSyncCompression);
- }
+ }
};
CompressionExecutor->Post(lambda);
-}
-
-void TWriteSession::OnCompressed(TBlock&& block, bool isSyncCompression) {
- TMemoryUsageChange memoryUsage;
- if (!isSyncCompression) {
- with_lock(Lock) {
- memoryUsage = OnCompressedImpl(std::move(block));
- }
- } else {
- memoryUsage = OnCompressedImpl(std::move(block));
- }
- if (memoryUsage.NowOk && !memoryUsage.WasOk) {
- EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
- }
-}
-
-//Called under lock or synchronously if compression is sync
-TMemoryUsageChange TWriteSession::OnCompressedImpl(TBlock&& block) {
- UpdateTimedCountersImpl();
+}
+
+void TWriteSession::OnCompressed(TBlock&& block, bool isSyncCompression) {
+ TMemoryUsageChange memoryUsage;
+ if (!isSyncCompression) {
+ with_lock(Lock) {
+ memoryUsage = OnCompressedImpl(std::move(block));
+ }
+ } else {
+ memoryUsage = OnCompressedImpl(std::move(block));
+ }
+ if (memoryUsage.NowOk && !memoryUsage.WasOk) {
+ EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
+ }
+}
+
+//Called under lock or synchronously if compression is sync
+TMemoryUsageChange TWriteSession::OnCompressedImpl(TBlock&& block) {
+ UpdateTimedCountersImpl();
Y_VERIFY(block.Valid);
- auto memoryUsage = OnMemoryUsageChangedImpl(static_cast<i64>(block.Data.size()) - block.OriginalMemoryUsage);
- (*Counters->BytesInflightUncompressed) -= block.OriginalSize;
- (*Counters->BytesInflightCompressed) += block.Data.size();
-
+ auto memoryUsage = OnMemoryUsageChangedImpl(static_cast<i64>(block.Data.size()) - block.OriginalMemoryUsage);
+ (*Counters->BytesInflightUncompressed) -= block.OriginalSize;
+ (*Counters->BytesInflightCompressed) += block.Data.size();
+
PackedMessagesToSend.emplace(std::move(block));
- SendImpl();
- return memoryUsage;
-}
-
-// Only called under lock
-void TWriteSession::ResetForRetryImpl() {
- SessionEstablished = false;
- const size_t totalPackedMessages = PackedMessagesToSend.size() + SentPackedMessage.size();
- const size_t totalOriginalMessages = OriginalMessagesToSend.size() + SentOriginalMessages.size();
- while (!SentPackedMessage.empty()) {
+ SendImpl();
+ return memoryUsage;
+}
+
+// Only called under lock
+void TWriteSession::ResetForRetryImpl() {
+ SessionEstablished = false;
+ const size_t totalPackedMessages = PackedMessagesToSend.size() + SentPackedMessage.size();
+ const size_t totalOriginalMessages = OriginalMessagesToSend.size() + SentOriginalMessages.size();
+ while (!SentPackedMessage.empty()) {
PackedMessagesToSend.emplace(std::move(SentPackedMessage.front()));
- SentPackedMessage.pop();
- }
- ui64 minSeqNo = PackedMessagesToSend.empty() ? LastSeqNo + 1 : PackedMessagesToSend.top().Offset;
+ SentPackedMessage.pop();
+ }
+ ui64 minSeqNo = PackedMessagesToSend.empty() ? LastSeqNo + 1 : PackedMessagesToSend.top().Offset;
std::queue<TOriginalMessage> freshOriginalMessagesToSend;
OriginalMessagesToSend.swap(freshOriginalMessagesToSend);
- while (!SentOriginalMessages.empty()) {
+ while (!SentOriginalMessages.empty()) {
OriginalMessagesToSend.emplace(std::move(SentOriginalMessages.front()));
- SentOriginalMessages.pop();
- }
- while (!freshOriginalMessagesToSend.empty()) {
+ SentOriginalMessages.pop();
+ }
+ while (!freshOriginalMessagesToSend.empty()) {
OriginalMessagesToSend.emplace(std::move(freshOriginalMessagesToSend.front()));
- freshOriginalMessagesToSend.pop();
- }
- if (!OriginalMessagesToSend.empty() && OriginalMessagesToSend.front().SeqNo < minSeqNo)
- minSeqNo = OriginalMessagesToSend.front().SeqNo;
- MinUnsentSeqNo = minSeqNo;
- Y_VERIFY(PackedMessagesToSend.size() == totalPackedMessages);
- Y_VERIFY(OriginalMessagesToSend.size() == totalOriginalMessages);
-}
-
-// Called from client Write() methods. With lock
-void TWriteSession::FlushWriteIfRequiredImpl() {
-
- if (!CurrentBatch.Empty() && !CurrentBatch.FlushRequested) {
- MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire());
- if (TInstant::Now() - CurrentBatch.StartedAt >= Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero())
- || CurrentBatch.CurrentSize >= Settings.BatchFlushSizeBytes_.GetOrElse(0)
- || CurrentBatch.CurrentSize >= MaxBlockSize
- || CurrentBatch.Messages.size() >= MaxBlockMessageCount
+ freshOriginalMessagesToSend.pop();
+ }
+ if (!OriginalMessagesToSend.empty() && OriginalMessagesToSend.front().SeqNo < minSeqNo)
+ minSeqNo = OriginalMessagesToSend.front().SeqNo;
+ MinUnsentSeqNo = minSeqNo;
+ Y_VERIFY(PackedMessagesToSend.size() == totalPackedMessages);
+ Y_VERIFY(OriginalMessagesToSend.size() == totalOriginalMessages);
+}
+
+// Called from client Write() methods. With lock
+void TWriteSession::FlushWriteIfRequiredImpl() {
+
+ if (!CurrentBatch.Empty() && !CurrentBatch.FlushRequested) {
+ MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire());
+ if (TInstant::Now() - CurrentBatch.StartedAt >= Settings.BatchFlushInterval_.GetOrElse(TDuration::Zero())
+ || CurrentBatch.CurrentSize >= Settings.BatchFlushSizeBytes_.GetOrElse(0)
+ || CurrentBatch.CurrentSize >= MaxBlockSize
+ || CurrentBatch.Messages.size() >= MaxBlockMessageCount
|| CurrentBatch.HasCodec()
- ) {
- WriteBatchImpl();
- return;
- }
- }
-}
-
-
-// Involves compression, but still called under lock.
-size_t TWriteSession::WriteBatchImpl() {
+ ) {
+ WriteBatchImpl();
+ return;
+ }
+ }
+}
+
+
+// Involves compression, but still called under lock.
+size_t TWriteSession::WriteBatchImpl() {
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "write " << CurrentBatch.Messages.size() << " messages with seqNo from "
- << CurrentBatch.Messages.begin()->SeqNo << " to " << CurrentBatch.Messages.back().SeqNo;
-
- Y_VERIFY(CurrentBatch.Messages.size() <= MaxBlockMessageCount);
-
+ << CurrentBatch.Messages.begin()->SeqNo << " to " << CurrentBatch.Messages.back().SeqNo;
+
+ Y_VERIFY(CurrentBatch.Messages.size() <= MaxBlockMessageCount);
+
const bool skipCompression = Settings.Codec_ == ECodec::RAW || CurrentBatch.HasCodec();
- if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) {
- MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire());
- }
-
- size_t size = 0;
- for (size_t i = 0; i != CurrentBatch.Messages.size();) {
- TBlock block{};
- for (; block.OriginalSize < MaxBlockSize && i != CurrentBatch.Messages.size(); ++i) {
- auto sequenceNumber = CurrentBatch.Messages[i].SeqNo;
- auto createTs = CurrentBatch.Messages[i].CreatedAt;
-
- if (!block.MessageCount) {
- block.Offset = sequenceNumber;
- }
-
- block.MessageCount += 1;
- const auto& datum = CurrentBatch.Messages[i].DataRef;
- block.OriginalSize += datum.size();
- block.OriginalMemoryUsage = CurrentBatch.Data.size();
- block.OriginalDataRefs.emplace_back(datum);
+ if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) {
+ MessagesAcquired += static_cast<ui64>(CurrentBatch.Acquire());
+ }
+
+ size_t size = 0;
+ for (size_t i = 0; i != CurrentBatch.Messages.size();) {
+ TBlock block{};
+ for (; block.OriginalSize < MaxBlockSize && i != CurrentBatch.Messages.size(); ++i) {
+ auto sequenceNumber = CurrentBatch.Messages[i].SeqNo;
+ auto createTs = CurrentBatch.Messages[i].CreatedAt;
+
+ if (!block.MessageCount) {
+ block.Offset = sequenceNumber;
+ }
+
+ block.MessageCount += 1;
+ const auto& datum = CurrentBatch.Messages[i].DataRef;
+ block.OriginalSize += datum.size();
+ block.OriginalMemoryUsage = CurrentBatch.Data.size();
+ block.OriginalDataRefs.emplace_back(datum);
if (CurrentBatch.Messages[i].Codec.Defined()) {
Y_VERIFY(CurrentBatch.Messages.size() == 1);
block.CodecID = GetCodecId(*CurrentBatch.Messages[i].Codec);
block.OriginalSize = CurrentBatch.Messages[i].OriginalSize;
block.Compressed = false;
}
- size += datum.size();
- UpdateTimedCountersImpl();
- (*Counters->BytesInflightUncompressed) += datum.size();
- (*Counters->MessagesInflight)++;
- OriginalMessagesToSend.emplace(sequenceNumber, createTs, datum.size());
- }
- block.Data = std::move(CurrentBatch.Data);
- if (skipCompression) {
+ size += datum.size();
+ UpdateTimedCountersImpl();
+ (*Counters->BytesInflightUncompressed) += datum.size();
+ (*Counters->MessagesInflight)++;
+ OriginalMessagesToSend.emplace(sequenceNumber, createTs, datum.size());
+ }
+ block.Data = std::move(CurrentBatch.Data);
+ if (skipCompression) {
PackedMessagesToSend.emplace(std::move(block));
- } else {
- CompressImpl(std::move(block));
- }
- }
- CurrentBatch.Reset();
- if (skipCompression) {
- SendImpl();
- }
- return size;
-}
-
-size_t GetMaxGrpcMessageSize() {
- return 120 * 1024 * 1024;
-}
-
-bool TWriteSession::IsReadyToSendNextImpl() const {
- if (!SessionEstablished) {
- return false;
- }
+ } else {
+ CompressImpl(std::move(block));
+ }
+ }
+ CurrentBatch.Reset();
+ if (skipCompression) {
+ SendImpl();
+ }
+ return size;
+}
+
+size_t GetMaxGrpcMessageSize() {
+ return 120 * 1024 * 1024;
+}
+
+bool TWriteSession::IsReadyToSendNextImpl() const {
+ if (!SessionEstablished) {
+ return false;
+ }
if (Aborting)
return false;
- if (PackedMessagesToSend.empty()) {
- return false;
- }
- Y_VERIFY(!OriginalMessagesToSend.empty(), "There are packed messages but no original messages");
- Y_VERIFY(OriginalMessagesToSend.front().SeqNo <= PackedMessagesToSend.top().Offset, "Lost original message(s)");
-
- return PackedMessagesToSend.top().Offset == OriginalMessagesToSend.front().SeqNo;
-}
-
+ if (PackedMessagesToSend.empty()) {
+ return false;
+ }
+ Y_VERIFY(!OriginalMessagesToSend.empty(), "There are packed messages but no original messages");
+ Y_VERIFY(OriginalMessagesToSend.front().SeqNo <= PackedMessagesToSend.top().Offset, "Lost original message(s)");
+
+ return PackedMessagesToSend.top().Offset == OriginalMessagesToSend.front().SeqNo;
+}
+
void TWriteSession::UpdateTokenIfNeededImpl() {
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: try to update token";
@@ -1009,120 +1009,120 @@ void TWriteSession::UpdateTokenIfNeededImpl() {
Processor->Write(std::move(clientMessage));
}
-void TWriteSession::SendImpl() {
- // External cycle splits ready blocks into multiple gRPC messages. Current gRPC message size hard limit is 64MiB
- while(IsReadyToSendNextImpl()) {
- TClientMessage clientMessage;
- auto* writeRequest = clientMessage.mutable_write_request();
- auto sentAtMs = TInstant::Now().MilliSeconds();
-
- // Sent blocks while we can without messages reordering
- while (IsReadyToSendNextImpl() && clientMessage.ByteSizeLong() < GetMaxGrpcMessageSize()) {
+void TWriteSession::SendImpl() {
+ // External cycle splits ready blocks into multiple gRPC messages. Current gRPC message size hard limit is 64MiB
+ while(IsReadyToSendNextImpl()) {
+ TClientMessage clientMessage;
+ auto* writeRequest = clientMessage.mutable_write_request();
+ auto sentAtMs = TInstant::Now().MilliSeconds();
+
+ // Sent blocks while we can without messages reordering
+ while (IsReadyToSendNextImpl() && clientMessage.ByteSizeLong() < GetMaxGrpcMessageSize()) {
const auto& block = PackedMessagesToSend.top();
Y_VERIFY(block.Valid);
- for (size_t i = 0; i != block.MessageCount; ++i) {
- Y_VERIFY(!OriginalMessagesToSend.empty());
-
- auto& message = OriginalMessagesToSend.front();
-
- writeRequest->add_sent_at_ms(sentAtMs);
- writeRequest->add_sequence_numbers(message.SeqNo + SeqNoShift);
- writeRequest->add_message_sizes(message.Size);
- writeRequest->add_created_at_ms(message.CreatedAt.MilliSeconds());
-
+ for (size_t i = 0; i != block.MessageCount; ++i) {
+ Y_VERIFY(!OriginalMessagesToSend.empty());
+
+ auto& message = OriginalMessagesToSend.front();
+
+ writeRequest->add_sent_at_ms(sentAtMs);
+ writeRequest->add_sequence_numbers(message.SeqNo + SeqNoShift);
+ writeRequest->add_message_sizes(message.Size);
+ writeRequest->add_created_at_ms(message.CreatedAt.MilliSeconds());
+
SentOriginalMessages.emplace(std::move(message));
- OriginalMessagesToSend.pop();
- }
-
- writeRequest->add_blocks_offsets(block.Offset);
- writeRequest->add_blocks_message_counts(block.MessageCount);
- writeRequest->add_blocks_part_numbers(block.PartNumber);
- writeRequest->add_blocks_uncompressed_sizes(block.OriginalSize);
- writeRequest->add_blocks_headers(block.CodecID);
- if (block.Compressed)
- writeRequest->add_blocks_data(block.Data.data(), block.Data.size());
- else {
- for (auto& buffer: block.OriginalDataRefs) {
- writeRequest->add_blocks_data(buffer.data(), buffer.size());
- }
- }
-
+ OriginalMessagesToSend.pop();
+ }
+
+ writeRequest->add_blocks_offsets(block.Offset);
+ writeRequest->add_blocks_message_counts(block.MessageCount);
+ writeRequest->add_blocks_part_numbers(block.PartNumber);
+ writeRequest->add_blocks_uncompressed_sizes(block.OriginalSize);
+ writeRequest->add_blocks_headers(block.CodecID);
+ if (block.Compressed)
+ writeRequest->add_blocks_data(block.Data.data(), block.Data.size());
+ else {
+ for (auto& buffer: block.OriginalDataRefs) {
+ writeRequest->add_blocks_data(buffer.data(), buffer.size());
+ }
+ }
+
TBlock moveBlock;
moveBlock.Move(block);
SentPackedMessage.emplace(std::move(moveBlock));
- PackedMessagesToSend.pop();
- }
+ PackedMessagesToSend.pop();
+ }
UpdateTokenIfNeededImpl();
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Send " << writeRequest->sequence_numbers_size() << " message(s) ("
- << OriginalMessagesToSend.size() << " left), first sequence number is "
- << writeRequest->sequence_numbers(0);
- Processor->Write(std::move(clientMessage));
- }
-}
-
-// Client method, no Lock
-bool TWriteSession::Close(TDuration closeTimeout) {
- if (AtomicGet(Aborting))
+ << OriginalMessagesToSend.size() << " left), first sequence number is "
+ << writeRequest->sequence_numbers(0);
+ Processor->Write(std::move(clientMessage));
+ }
+}
+
+// Client method, no Lock
+bool TWriteSession::Close(TDuration closeTimeout) {
+ if (AtomicGet(Aborting))
return false;
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session: close. Timeout = " << closeTimeout.MilliSeconds() << " ms";
- auto startTime = TInstant::Now();
- auto remaining = closeTimeout;
- bool ready = false;
- bool needSetSeqNoValue = false;
- while (remaining > TDuration::Zero()) {
- with_lock(Lock) {
- if (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) {
- ready = true;
- }
+ auto startTime = TInstant::Now();
+ auto remaining = closeTimeout;
+ bool ready = false;
+ bool needSetSeqNoValue = false;
+ while (remaining > TDuration::Zero()) {
+ with_lock(Lock) {
+ if (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) {
+ ready = true;
+ }
if (AtomicGet(Aborting))
break;
- }
- if (ready) {
- break;
- }
- remaining = closeTimeout - (TInstant::Now() - startTime);
- Sleep(Min(TDuration::MilliSeconds(100), remaining));
- }
- with_lock(Lock) {
+ }
+ if (ready) {
+ break;
+ }
+ remaining = closeTimeout - (TInstant::Now() - startTime);
+ Sleep(Min(TDuration::MilliSeconds(100), remaining));
+ }
+ with_lock(Lock) {
ready = (OriginalMessagesToSend.empty() && SentOriginalMessages.empty()) && !AtomicGet(Aborting);
- }
+ }
with_lock(Lock) {
CloseImpl(EStatus::SUCCESS, NYql::TIssues{});
- needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true);
+ needSetSeqNoValue = !InitSeqNoSetDone && (InitSeqNoSetDone = true);
+ }
+ if (needSetSeqNoValue) {
+ InitSeqNoPromise.SetException("session closed");
}
- if (needSetSeqNoValue) {
- InitSeqNoPromise.SetException("session closed");
- }
if (ready) {
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session: gracefully shut down, all writes complete";
} else {
DbDriverState->Log << TLOG_WARNING << LogPrefix() << "Write session: could not confirm all writes in time or session aborted, perform hard shutdown";
}
- return ready;
-}
-
-void TWriteSession::HandleWakeUpImpl() {
- FlushWriteIfRequiredImpl();
- if (AtomicGet(Aborting)) {
- return;
- }
- auto callback = [weakThis = this->weak_from_this()] (bool ok)
- {
- if (!ok)
- return;
- if (auto sharedThis = weakThis.lock()) {
- with_lock(sharedThis->Lock) {
- sharedThis->HandleWakeUpImpl();
- }
- }
- };
- auto enqueueTokenCallback = [weakThis = this->weak_from_this()] (bool ok) {
- if (!ok)
- return;
- if (auto sharedThis = weakThis.lock()) {
- sharedThis->EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
- }
- };
+ return ready;
+}
+
+void TWriteSession::HandleWakeUpImpl() {
+ FlushWriteIfRequiredImpl();
+ if (AtomicGet(Aborting)) {
+ return;
+ }
+ auto callback = [weakThis = this->weak_from_this()] (bool ok)
+ {
+ if (!ok)
+ return;
+ if (auto sharedThis = weakThis.lock()) {
+ with_lock(sharedThis->Lock) {
+ sharedThis->HandleWakeUpImpl();
+ }
+ }
+ };
+ auto enqueueTokenCallback = [weakThis = this->weak_from_this()] (bool ok) {
+ if (!ok)
+ return;
+ if (auto sharedThis = weakThis.lock()) {
+ sharedThis->EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{TContinuationToken{}});
+ }
+ };
if (TInstant::Now() - LastTokenUpdate > UPDATE_TOKEN_PERIOD) {
LastTokenUpdate = TInstant::Now();
UpdateTokenIfNeededImpl();
@@ -1132,179 +1132,179 @@ void TWriteSession::HandleWakeUpImpl() {
? WakeupInterval
: WakeupInterval - Min(Now() - CurrentBatch.StartedAt, WakeupInterval);
Connections->ScheduleCallback(flushAfter, std::move(callback));
-}
-
-void TWriteSession::UpdateTimedCountersImpl() {
- auto now = TInstant::Now();
- auto delta = (now - LastCountersUpdateTs).MilliSeconds();
- double percent = 100.0 / Settings.MaxMemoryUsage_;
-
- Counters->TotalBytesInflightUsageByTime->Collect(*Counters->BytesInflightTotal * percent, delta);
- Counters->UncompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightUncompressed * percent, delta);
- Counters->CompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightCompressed * percent, delta);
-
- *Counters->CurrentSessionLifetimeMs = (TInstant::Now() - SessionStartedTs).MilliSeconds();
- LastCountersUpdateTs = now;
- if (LastCountersLogTs == TInstant::Zero() || TInstant::Now() - LastCountersLogTs > TDuration::Seconds(60)) {
- LastCountersLogTs = TInstant::Now();
-
-#define LOG_COUNTER(counter) \
- << " " Y_STRINGIZE(counter) ": " \
- << Counters->counter->Val() \
- /**/
-
+}
+
+void TWriteSession::UpdateTimedCountersImpl() {
+ auto now = TInstant::Now();
+ auto delta = (now - LastCountersUpdateTs).MilliSeconds();
+ double percent = 100.0 / Settings.MaxMemoryUsage_;
+
+ Counters->TotalBytesInflightUsageByTime->Collect(*Counters->BytesInflightTotal * percent, delta);
+ Counters->UncompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightUncompressed * percent, delta);
+ Counters->CompressedBytesInflightUsageByTime->Collect(*Counters->BytesInflightCompressed * percent, delta);
+
+ *Counters->CurrentSessionLifetimeMs = (TInstant::Now() - SessionStartedTs).MilliSeconds();
+ LastCountersUpdateTs = now;
+ if (LastCountersLogTs == TInstant::Zero() || TInstant::Now() - LastCountersLogTs > TDuration::Seconds(60)) {
+ LastCountersLogTs = TInstant::Now();
+
+#define LOG_COUNTER(counter) \
+ << " " Y_STRINGIZE(counter) ": " \
+ << Counters->counter->Val() \
+ /**/
+
DbDriverState->Log << TLOG_INFO << LogPrefix()
- << "Counters: {"
- LOG_COUNTER(Errors)
- LOG_COUNTER(CurrentSessionLifetimeMs)
- LOG_COUNTER(BytesWritten)
- LOG_COUNTER(MessagesWritten)
- LOG_COUNTER(BytesWrittenCompressed)
- LOG_COUNTER(BytesInflightUncompressed)
- LOG_COUNTER(BytesInflightCompressed)
- LOG_COUNTER(BytesInflightTotal)
- LOG_COUNTER(MessagesInflight)
- << " }";
-
-#undef LOG_COUNTER
- }
-}
-
+ << "Counters: {"
+ LOG_COUNTER(Errors)
+ LOG_COUNTER(CurrentSessionLifetimeMs)
+ LOG_COUNTER(BytesWritten)
+ LOG_COUNTER(MessagesWritten)
+ LOG_COUNTER(BytesWrittenCompressed)
+ LOG_COUNTER(BytesInflightUncompressed)
+ LOG_COUNTER(BytesInflightCompressed)
+ LOG_COUNTER(BytesInflightTotal)
+ LOG_COUNTER(MessagesInflight)
+ << " }";
+
+#undef LOG_COUNTER
+ }
+}
+
void TWriteSession::AbortImpl() {
- if (!AtomicGet(Aborting)) {
+ if (!AtomicGet(Aborting)) {
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: aborting";
- AtomicSet(Aborting, 1);
- Cancel(ConnectContext);
- Cancel(ConnectTimeoutContext);
- Cancel(ConnectDelayContext);
- ///Cancel(ClientContext);
- if (Processor)
- Processor->Cancel();
- }
-}
-
-void TWriteSession::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) {
+ AtomicSet(Aborting, 1);
+ Cancel(ConnectContext);
+ Cancel(ConnectTimeoutContext);
+ Cancel(ConnectDelayContext);
+ ///Cancel(ClientContext);
+ if (Processor)
+ Processor->Cancel();
+ }
+}
+
+void TWriteSession::CloseImpl(EStatus statusCode, NYql::TIssues&& issues) {
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will now close";
EventsQueue->Close(TSessionClosedEvent(statusCode, std::move(issues)));
- AbortImpl();
-}
-
-void TWriteSession::CloseImpl(EStatus statusCode, const TString& message) {
- NYql::TIssues issues;
- issues.AddIssue(message);
- CloseImpl(statusCode, std::move(issues));
-}
-
-void TWriteSession::CloseImpl(TPlainStatus&& status) {
+ AbortImpl();
+}
+
+void TWriteSession::CloseImpl(EStatus statusCode, const TString& message) {
+ NYql::TIssues issues;
+ issues.AddIssue(message);
+ CloseImpl(statusCode, std::move(issues));
+}
+
+void TWriteSession::CloseImpl(TPlainStatus&& status) {
DbDriverState->Log << TLOG_INFO << LogPrefix() << "Write session will now close";
EventsQueue->Close(TSessionClosedEvent(std::move(status)));
- AbortImpl();
-}
-
-TWriteSession::~TWriteSession() {
+ AbortImpl();
+}
+
+TWriteSession::~TWriteSession() {
DbDriverState->Log << TLOG_DEBUG << LogPrefix() << "Write session: destroy";
- bool needClose = false;
- with_lock(Lock) {
- if (!AtomicGet(Aborting)) {
- CloseImpl(EStatus::SUCCESS, NYql::TIssues{});
-
- needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true);
- }
- }
- if (needClose)
- InitSeqNoPromise.SetException("session closed");
-}
-
-TSimpleBlockingWriteSession::TSimpleBlockingWriteSession(
- const TWriteSessionSettings& settings,
+ bool needClose = false;
+ with_lock(Lock) {
+ if (!AtomicGet(Aborting)) {
+ CloseImpl(EStatus::SUCCESS, NYql::TIssues{});
+
+ needClose = !InitSeqNoSetDone && (InitSeqNoSetDone = true);
+ }
+ }
+ if (needClose)
+ InitSeqNoPromise.SetException("session closed");
+}
+
+TSimpleBlockingWriteSession::TSimpleBlockingWriteSession(
+ const TWriteSessionSettings& settings,
std::shared_ptr<TPersQueueClient::TImpl> client,
- std::shared_ptr<TGRpcConnectionsImpl> connections,
- TDbDriverStatePtr dbDriverState
-) {
- auto alteredSettings = settings;
- alteredSettings.EventHandlers_.AcksHandler_ = [this](TWriteSessionEvent::TAcksEvent& event) {this->HandleAck(event); };
- alteredSettings.EventHandlers_.ReadyToAcceptHander_ = [this](TWriteSessionEvent::TReadyToAcceptEvent& event)
- {this->HandleReady(event); };
- alteredSettings.EventHandlers_.SessionClosedHandler_ = [this](const TSessionClosedEvent& event) {this->HandleClosed(event); };
-
- Writer = std::make_shared<TWriteSession>(
+ std::shared_ptr<TGRpcConnectionsImpl> connections,
+ TDbDriverStatePtr dbDriverState
+) {
+ auto alteredSettings = settings;
+ alteredSettings.EventHandlers_.AcksHandler_ = [this](TWriteSessionEvent::TAcksEvent& event) {this->HandleAck(event); };
+ alteredSettings.EventHandlers_.ReadyToAcceptHander_ = [this](TWriteSessionEvent::TReadyToAcceptEvent& event)
+ {this->HandleReady(event); };
+ alteredSettings.EventHandlers_.SessionClosedHandler_ = [this](const TSessionClosedEvent& event) {this->HandleClosed(event); };
+
+ Writer = std::make_shared<TWriteSession>(
alteredSettings, client, connections, dbDriverState
- );
- Writer->Start(TDuration::Max());
-}
-
-ui64 TSimpleBlockingWriteSession::GetInitSeqNo() {
- return Writer->GetInitSeqNo().GetValueSync();
-}
-
-bool TSimpleBlockingWriteSession::Write(
- TStringBuf data, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp, const TDuration& blockTimeout
-) {
+ );
+ Writer->Start(TDuration::Max());
+}
+
+ui64 TSimpleBlockingWriteSession::GetInitSeqNo() {
+ return Writer->GetInitSeqNo().GetValueSync();
+}
+
+bool TSimpleBlockingWriteSession::Write(
+ TStringBuf data, TMaybe<ui64> seqNo, TMaybe<TInstant> createTimestamp, const TDuration& blockTimeout
+) {
if (!IsAlive())
return false;
- auto continuationToken = WaitForToken(blockTimeout);
- if (continuationToken.Defined()) {
- Writer->Write(std::move(*continuationToken), std::move(data), seqNo, createTimestamp);
- return true;
- }
- return false;
-}
-
-TMaybe<TContinuationToken> TSimpleBlockingWriteSession::WaitForToken(const TDuration& timeout) {
- auto startTime = TInstant::Now();
- TDuration remainingTime = timeout;
- TMaybe<TContinuationToken> token = Nothing();
- while(!token.Defined() && remainingTime > TDuration::Zero()) {
- with_lock(Lock) {
- if (!ContinueTokens.empty()) {
- token = std::move(ContinueTokens.front());
- ContinueTokens.pop();
- }
- }
+ auto continuationToken = WaitForToken(blockTimeout);
+ if (continuationToken.Defined()) {
+ Writer->Write(std::move(*continuationToken), std::move(data), seqNo, createTimestamp);
+ return true;
+ }
+ return false;
+}
+
+TMaybe<TContinuationToken> TSimpleBlockingWriteSession::WaitForToken(const TDuration& timeout) {
+ auto startTime = TInstant::Now();
+ TDuration remainingTime = timeout;
+ TMaybe<TContinuationToken> token = Nothing();
+ while(!token.Defined() && remainingTime > TDuration::Zero()) {
+ with_lock(Lock) {
+ if (!ContinueTokens.empty()) {
+ token = std::move(ContinueTokens.front());
+ ContinueTokens.pop();
+ }
+ }
if (!IsAlive())
return Nothing();
- if (token.Defined()) {
- return std::move(*token);
- }
- else {
- remainingTime = timeout - (TInstant::Now() - startTime);
- Sleep(Min(remainingTime, TDuration::MilliSeconds(100)));
- }
- }
- return Nothing();
-}
-
-TWriterCounters::TPtr TSimpleBlockingWriteSession::GetCounters() {
- return Writer->GetCounters();
-}
-
-
-bool TSimpleBlockingWriteSession::IsAlive() const {
- bool closed = false;
- with_lock(Lock) {
- closed = Closed;
- }
- return !closed;
-}
-
-void TSimpleBlockingWriteSession::HandleAck(TWriteSessionEvent::TAcksEvent& event) {
- Y_UNUSED(event);
-}
-
-void TSimpleBlockingWriteSession::HandleReady(TWriteSessionEvent::TReadyToAcceptEvent& event) {
- with_lock(Lock) {
+ if (token.Defined()) {
+ return std::move(*token);
+ }
+ else {
+ remainingTime = timeout - (TInstant::Now() - startTime);
+ Sleep(Min(remainingTime, TDuration::MilliSeconds(100)));
+ }
+ }
+ return Nothing();
+}
+
+TWriterCounters::TPtr TSimpleBlockingWriteSession::GetCounters() {
+ return Writer->GetCounters();
+}
+
+
+bool TSimpleBlockingWriteSession::IsAlive() const {
+ bool closed = false;
+ with_lock(Lock) {
+ closed = Closed;
+ }
+ return !closed;
+}
+
+void TSimpleBlockingWriteSession::HandleAck(TWriteSessionEvent::TAcksEvent& event) {
+ Y_UNUSED(event);
+}
+
+void TSimpleBlockingWriteSession::HandleReady(TWriteSessionEvent::TReadyToAcceptEvent& event) {
+ with_lock(Lock) {
ContinueTokens.emplace(std::move(event.ContinuationToken));
- }
-}
-void TSimpleBlockingWriteSession::HandleClosed(const TSessionClosedEvent&) {
- with_lock(Lock) {
- Closed = true;
- }
-}
-bool TSimpleBlockingWriteSession::Close(TDuration closeTimeout) {
- return Writer->Close(std::move(closeTimeout));
-}
-
-}; // namespace NYdb::NPersQueue
+ }
+}
+void TSimpleBlockingWriteSession::HandleClosed(const TSessionClosedEvent&) {
+ with_lock(Lock) {
+ Closed = true;
+ }
+}
+bool TSimpleBlockingWriteSession::Close(TDuration closeTimeout) {
+ return Writer->Close(std::move(closeTimeout));
+}
+
+}; // namespace NYdb::NPersQueue
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h
index 9b3d8f906e..324550bb02 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h
@@ -1,5 +1,5 @@
-#pragma once
-
+#pragma once
+
#include "common.h"
#include "persqueue_impl.h"
@@ -7,27 +7,27 @@
#include <util/generic/buffer.h>
-
-namespace NYdb::NPersQueue {
-
-inline const TString& GetCodecId(const ECodec codec) {
- static THashMap<ECodec, TString> idByCodec{
- {ECodec::RAW, TString(1, '\0')},
- {ECodec::GZIP, "\1"},
- {ECodec::LZOP, "\2"},
- {ECodec::ZSTD, "\3"}
- };
- Y_VERIFY(idByCodec.contains(codec));
- return idByCodec[codec];
-}
-
+
+namespace NYdb::NPersQueue {
+
+inline const TString& GetCodecId(const ECodec codec) {
+ static THashMap<ECodec, TString> idByCodec{
+ {ECodec::RAW, TString(1, '\0')},
+ {ECodec::GZIP, "\1"},
+ {ECodec::LZOP, "\2"},
+ {ECodec::ZSTD, "\3"}
+ };
+ Y_VERIFY(idByCodec.contains(codec));
+ return idByCodec[codec];
+}
+
class TWriteSessionEventsQueue : public TBaseSessionEventsQueue<TWriteSessionSettings, TWriteSessionEvent::TEvent> {
using TParent = TBaseSessionEventsQueue<TWriteSessionSettings, TWriteSessionEvent::TEvent>;
-public:
- TWriteSessionEventsQueue(const TWriteSessionSettings& settings)
- : TParent(settings)
- {}
-
+public:
+ TWriteSessionEventsQueue(const TWriteSessionSettings& settings)
+ : TParent(settings)
+ {}
+
void PushEvent(TEventInfo eventInfo) {
if (Closed || ApplyHandler(eventInfo)) {
return;
@@ -98,33 +98,33 @@ public:
waiter.Signal();
}
-private:
+private:
struct THandlersVisitor : public TParent::TBaseHandlersVisitor {
using TParent::TBaseHandlersVisitor::TBaseHandlersVisitor;
-#define DECLARE_HANDLER(type, handler, answer) \
- bool operator()(type& event) { \
- if (Settings.EventHandlers_.handler) { \
- Settings.EventHandlers_.handler(event); \
- return answer; \
- } \
- return false; \
- } \
- /**/
- DECLARE_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler_, true);
- DECLARE_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHander_, true);
- DECLARE_HANDLER(TSessionClosedEvent, SessionClosedHandler_, false); // Not applied
-
-#undef DECLARE_HANDLER
+#define DECLARE_HANDLER(type, handler, answer) \
+ bool operator()(type& event) { \
+ if (Settings.EventHandlers_.handler) { \
+ Settings.EventHandlers_.handler(event); \
+ return answer; \
+ } \
+ return false; \
+ } \
+ /**/
+ DECLARE_HANDLER(TWriteSessionEvent::TAcksEvent, AcksHandler_, true);
+ DECLARE_HANDLER(TWriteSessionEvent::TReadyToAcceptEvent, ReadyToAcceptHander_, true);
+ DECLARE_HANDLER(TSessionClosedEvent, SessionClosedHandler_, false); // Not applied
+
+#undef DECLARE_HANDLER
bool Visit() {
- return std::visit(*this, EventInfo.Event);
- }
-
- };
-
+ return std::visit(*this, EventInfo.Event);
+ }
+
+ };
+
bool ApplyHandler(TEventInfo& eventInfo) {
THandlersVisitor visitor(Settings, eventInfo);
- return visitor.Visit();
- }
+ return visitor.Visit();
+ }
TEventInfo GetEventImpl() { // Assumes that we're under lock and that the event queue has events.
Y_ASSERT(HasEventsImpl());
@@ -137,98 +137,98 @@ private:
Y_ASSERT(CloseEvent);
return {*CloseEvent};
}
-};
-
-struct TMemoryUsageChange {
- bool WasOk; //!< MemoryUsage <= Config.MaxMemoryUsage_ before update
- bool NowOk; //!< Same, only after update
-};
-
-namespace NTests {
- class TSimpleWriteSessionTestAdapter;
-}
-
-class TWriteSession : public IWriteSession,
+};
+
+struct TMemoryUsageChange {
+ bool WasOk; //!< MemoryUsage <= Config.MaxMemoryUsage_ before update
+ bool NowOk; //!< Same, only after update
+};
+
+namespace NTests {
+ class TSimpleWriteSessionTestAdapter;
+}
+
+class TWriteSession : public IWriteSession,
public std::enable_shared_from_this<TWriteSession> {
-private:
- friend class TSimpleBlockingWriteSession;
+private:
+ friend class TSimpleBlockingWriteSession;
friend class TPersQueueClient;
- friend class NTests::TSimpleWriteSessionTestAdapter;
-
- using TClientMessage = Ydb::PersQueue::V1::StreamingWriteClientMessage;
- using TServerMessage = Ydb::PersQueue::V1::StreamingWriteServerMessage;
- using IWriteSessionConnectionProcessorFactory =
+ friend class NTests::TSimpleWriteSessionTestAdapter;
+
+ using TClientMessage = Ydb::PersQueue::V1::StreamingWriteClientMessage;
+ using TServerMessage = Ydb::PersQueue::V1::StreamingWriteServerMessage;
+ using IWriteSessionConnectionProcessorFactory =
TPersQueueClient::TImpl::IWriteSessionConnectionProcessorFactory;
- using IProcessor = IWriteSessionConnectionProcessorFactory::IProcessor;
-
- struct TMessage {
- ui64 SeqNo;
- TInstant CreatedAt;
- TStringBuf DataRef;
+ using IProcessor = IWriteSessionConnectionProcessorFactory::IProcessor;
+
+ struct TMessage {
+ ui64 SeqNo;
+ TInstant CreatedAt;
+ TStringBuf DataRef;
TMaybe<ECodec> Codec;
ui32 OriginalSize; // only for coded messages
TMessage(ui64 seqNo, const TInstant& createdAt, TStringBuf data, TMaybe<ECodec> codec = {}, ui32 originalSize = 0)
- : SeqNo(seqNo)
- , CreatedAt(createdAt)
- , DataRef(data)
+ : SeqNo(seqNo)
+ , CreatedAt(createdAt)
+ , DataRef(data)
, Codec(codec)
, OriginalSize(originalSize)
- {}
- };
-
- struct TMessageBatch {
- TBuffer Data;
- TVector<TMessage> Messages;
- ui64 CurrentSize = 0;
- TInstant StartedAt = TInstant::Zero();
- bool Acquired = false;
- bool FlushRequested = false;
+ {}
+ };
+
+ struct TMessageBatch {
+ TBuffer Data;
+ TVector<TMessage> Messages;
+ ui64 CurrentSize = 0;
+ TInstant StartedAt = TInstant::Zero();
+ bool Acquired = false;
+ bool FlushRequested = false;
void Add(ui64 seqNo, const TInstant& createdAt, TStringBuf data, TMaybe<ECodec> codec, ui32 originalSize) {
- if (StartedAt == TInstant::Zero())
- StartedAt = TInstant::Now();
+ if (StartedAt == TInstant::Zero())
+ StartedAt = TInstant::Now();
CurrentSize += codec ? originalSize : data.size();
Messages.emplace_back(seqNo, createdAt, data, codec, originalSize);
- Acquired = false;
- }
-
+ Acquired = false;
+ }
+
bool HasCodec() const {
return Messages.empty() ? false : Messages.front().Codec.Defined();
}
- bool Acquire() {
- if (Acquired || Messages.empty())
- return false;
- auto currSize = Data.size();
- Data.Append(Messages.back().DataRef.data(), Messages.back().DataRef.size());
- Messages.back().DataRef = TStringBuf(Data.data() + currSize, Data.size() - currSize);
- Acquired = true;
- return true;
- }
-
- bool Empty() const noexcept {
- return CurrentSize == 0 && Messages.empty();
- }
-
- void Reset() {
- StartedAt = TInstant::Zero();
- Messages.clear();
- Data.Clear();
- Acquired = false;
- CurrentSize = 0;
- FlushRequested = false;
- }
- };
-
- struct TBlock {
- size_t Offset = 0; //!< First message sequence number in the block
- size_t MessageCount = 0;
- size_t PartNumber = 0;
- size_t OriginalSize = 0;
- size_t OriginalMemoryUsage = 0;
- TString CodecID = GetCodecId(ECodec::RAW);
+ bool Acquire() {
+ if (Acquired || Messages.empty())
+ return false;
+ auto currSize = Data.size();
+ Data.Append(Messages.back().DataRef.data(), Messages.back().DataRef.size());
+ Messages.back().DataRef = TStringBuf(Data.data() + currSize, Data.size() - currSize);
+ Acquired = true;
+ return true;
+ }
+
+ bool Empty() const noexcept {
+ return CurrentSize == 0 && Messages.empty();
+ }
+
+ void Reset() {
+ StartedAt = TInstant::Zero();
+ Messages.clear();
+ Data.Clear();
+ Acquired = false;
+ CurrentSize = 0;
+ FlushRequested = false;
+ }
+ };
+
+ struct TBlock {
+ size_t Offset = 0; //!< First message sequence number in the block
+ size_t MessageCount = 0;
+ size_t PartNumber = 0;
+ size_t OriginalSize = 0;
+ size_t OriginalMemoryUsage = 0;
+ TString CodecID = GetCodecId(ECodec::RAW);
mutable TVector<TStringBuf> OriginalDataRefs;
mutable TBuffer Data;
- bool Compressed = false;
+ bool Compressed = false;
mutable bool Valid = true;
TBlock& operator=(TBlock&&) = default;
@@ -250,69 +250,69 @@ private:
rhs.Data.Clear();
rhs.OriginalDataRefs.clear();
}
- };
-
- struct TOriginalMessage {
- ui64 SeqNo;
- TInstant CreatedAt;
- size_t Size;
- TOriginalMessage(const ui64 sequenceNumber, const TInstant createdAt, const size_t size)
- : SeqNo(sequenceNumber)
- , CreatedAt(createdAt)
- , Size(size)
- {}
- };
-
- //! Block comparer, makes block with smallest offset (first sequence number) appear on top of the PackedMessagesToSend priority queue
- struct Greater {
- bool operator() (const TBlock& lhs, const TBlock& rhs) {
- return lhs.Offset > rhs.Offset;
- }
- };
-
- struct THandleResult {
- bool DoRestart = false;
- TDuration StartDelay = TDuration::Zero();
- bool DoStop = false;
- bool DoSetSeqNo = false;
- };
- struct TProcessSrvMessageResult {
- THandleResult HandleResult;
- TMaybe<ui64> InitSeqNo;
- TVector<TWriteSessionEvent::TEvent> Events;
- bool Ok = true;
- };
-
- THandleResult OnErrorImpl(NYdb::TPlainStatus&& status); // true - should Start(), false - should Close(), empty - no action
-
-public:
- TWriteSession(const TWriteSessionSettings& settings,
+ };
+
+ struct TOriginalMessage {
+ ui64 SeqNo;
+ TInstant CreatedAt;
+ size_t Size;
+ TOriginalMessage(const ui64 sequenceNumber, const TInstant createdAt, const size_t size)
+ : SeqNo(sequenceNumber)
+ , CreatedAt(createdAt)
+ , Size(size)
+ {}
+ };
+
+ //! Block comparer, makes block with smallest offset (first sequence number) appear on top of the PackedMessagesToSend priority queue
+ struct Greater {
+ bool operator() (const TBlock& lhs, const TBlock& rhs) {
+ return lhs.Offset > rhs.Offset;
+ }
+ };
+
+ struct THandleResult {
+ bool DoRestart = false;
+ TDuration StartDelay = TDuration::Zero();
+ bool DoStop = false;
+ bool DoSetSeqNo = false;
+ };
+ struct TProcessSrvMessageResult {
+ THandleResult HandleResult;
+ TMaybe<ui64> InitSeqNo;
+ TVector<TWriteSessionEvent::TEvent> Events;
+ bool Ok = true;
+ };
+
+ THandleResult OnErrorImpl(NYdb::TPlainStatus&& status); // true - should Start(), false - should Close(), empty - no action
+
+public:
+ TWriteSession(const TWriteSessionSettings& settings,
std::shared_ptr<TPersQueueClient::TImpl> client,
- std::shared_ptr<TGRpcConnectionsImpl> connections,
- TDbDriverStatePtr dbDriverState);
-
- TMaybe<TWriteSessionEvent::TEvent> GetEvent(bool block = false) override;
- TVector<TWriteSessionEvent::TEvent> GetEvents(bool block = false,
- TMaybe<size_t> maxEventsCount = Nothing()) override;
- NThreading::TFuture<ui64> GetInitSeqNo() override;
-
- void Write(TContinuationToken&& continuationToken, TStringBuf data,
- TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) override;
-
+ std::shared_ptr<TGRpcConnectionsImpl> connections,
+ TDbDriverStatePtr dbDriverState);
+
+ TMaybe<TWriteSessionEvent::TEvent> GetEvent(bool block = false) override;
+ TVector<TWriteSessionEvent::TEvent> GetEvents(bool block = false,
+ TMaybe<size_t> maxEventsCount = Nothing()) override;
+ NThreading::TFuture<ui64> GetInitSeqNo() override;
+
+ void Write(TContinuationToken&& continuationToken, TStringBuf data,
+ TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) override;
+
void WriteEncoded(TContinuationToken&& continuationToken, TStringBuf data, ECodec codec, ui32 originalSize,
TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) override;
- NThreading::TFuture<void> WaitEvent() override;
-
- // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration.
- bool Close(TDuration closeTimeout = TDuration::Max()) override;
-
+ NThreading::TFuture<void> WaitEvent() override;
+
+ // Empty maybe - block till all work is done. Otherwise block at most at closeTimeout duration.
+ bool Close(TDuration closeTimeout = TDuration::Max()) override;
+
TWriterCounters::TPtr GetCounters() override {Y_FAIL("Unimplemented"); } //ToDo - unimplemented;
-
- ~TWriteSession(); // will not call close - destroy everything without acks
-
-private:
+
+ ~TWriteSession(); // will not call close - destroy everything without acks
+
+private:
TString LogPrefix() const;
@@ -321,152 +321,152 @@ private:
void WriteInternal(TContinuationToken&& continuationToken, TStringBuf data, TMaybe<ECodec> codec, ui32 originalSize,
TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing());
- void FlushWriteIfRequiredImpl();
- size_t WriteBatchImpl();
- void Start(const TDuration& delay);
- void InitWriter();
-
- void DoCdsRequest(TDuration delay = TDuration::Zero());
- void OnCdsResponse(TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result);
- void OnConnect(TPlainStatus&& st, typename IProcessor::TPtr&& processor,
- const NGrpc::IQueueClientContextPtr& connectContext);
- void OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext);
- void ResetForRetryImpl();
- THandleResult RestartImpl(const TPlainStatus& status);
+ void FlushWriteIfRequiredImpl();
+ size_t WriteBatchImpl();
+ void Start(const TDuration& delay);
+ void InitWriter();
+
+ void DoCdsRequest(TDuration delay = TDuration::Zero());
+ void OnCdsResponse(TStatus& status, const Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult& result);
+ void OnConnect(TPlainStatus&& st, typename IProcessor::TPtr&& processor,
+ const NGrpc::IQueueClientContextPtr& connectContext);
+ void OnConnectTimeout(const NGrpc::IQueueClientContextPtr& connectTimeoutContext);
+ void ResetForRetryImpl();
+ THandleResult RestartImpl(const TPlainStatus& status);
void DoConnect(const TDuration& delay, const TString& endpoint);
- void InitImpl();
- void ReadFromProcessor(); // Assumes that we're under lock.
- void WriteToProcessorImpl(TClientMessage&& req); // Assumes that we're under lock.
- void OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration);
- void OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration);
- TProcessSrvMessageResult ProcessServerMessageImpl();
- TMemoryUsageChange OnMemoryUsageChangedImpl(i64 diff);
- void CompressImpl(TBlock&& block);
- void OnCompressed(TBlock&& block, bool isSyncCompression=false);
- TMemoryUsageChange OnCompressedImpl(TBlock&& block);
-
- //TString GetDebugIdentity() const;
- Ydb::PersQueue::V1::StreamingWriteClientMessage GetInitClientMessage();
- bool CleanupOnAcknowledged(ui64 sequenceNumber);
- bool IsReadyToSendNextImpl() const;
- ui64 GetNextSeqNoImpl(const TMaybe<ui64>& seqNo);
- void SendImpl();
+ void InitImpl();
+ void ReadFromProcessor(); // Assumes that we're under lock.
+ void WriteToProcessorImpl(TClientMessage&& req); // Assumes that we're under lock.
+ void OnReadDone(NGrpc::TGrpcStatus&& grpcStatus, size_t connectionGeneration);
+ void OnWriteDone(NGrpc::TGrpcStatus&& status, size_t connectionGeneration);
+ TProcessSrvMessageResult ProcessServerMessageImpl();
+ TMemoryUsageChange OnMemoryUsageChangedImpl(i64 diff);
+ void CompressImpl(TBlock&& block);
+ void OnCompressed(TBlock&& block, bool isSyncCompression=false);
+ TMemoryUsageChange OnCompressedImpl(TBlock&& block);
+
+ //TString GetDebugIdentity() const;
+ Ydb::PersQueue::V1::StreamingWriteClientMessage GetInitClientMessage();
+ bool CleanupOnAcknowledged(ui64 sequenceNumber);
+ bool IsReadyToSendNextImpl() const;
+ ui64 GetNextSeqNoImpl(const TMaybe<ui64>& seqNo);
+ void SendImpl();
void AbortImpl();
- void CloseImpl(EStatus statusCode, NYql::TIssues&& issues);
- void CloseImpl(EStatus statusCode, const TString& message);
- void CloseImpl(TPlainStatus&& status);
-
- void OnErrorResolved() {
- RetryState = nullptr;
- }
- void CheckHandleResultImpl(THandleResult& result);
- void ProcessHandleResult(THandleResult& result);
- void HandleWakeUpImpl();
- void UpdateTimedCountersImpl();
-
-private:
- TWriteSessionSettings Settings;
+ void CloseImpl(EStatus statusCode, NYql::TIssues&& issues);
+ void CloseImpl(EStatus statusCode, const TString& message);
+ void CloseImpl(TPlainStatus&& status);
+
+ void OnErrorResolved() {
+ RetryState = nullptr;
+ }
+ void CheckHandleResultImpl(THandleResult& result);
+ void ProcessHandleResult(THandleResult& result);
+ void HandleWakeUpImpl();
+ void UpdateTimedCountersImpl();
+
+private:
+ TWriteSessionSettings Settings;
std::shared_ptr<TPersQueueClient::TImpl> Client;
- std::shared_ptr<TGRpcConnectionsImpl> Connections;
- TString TargetCluster;
- TString InitialCluster;
- TString CurrentCluster;
- bool OnSeqNoShift = false;
+ std::shared_ptr<TGRpcConnectionsImpl> Connections;
+ TString TargetCluster;
+ TString InitialCluster;
+ TString CurrentCluster;
+ bool OnSeqNoShift = false;
TString PreferredClusterByCDS;
- std::shared_ptr<IWriteSessionConnectionProcessorFactory> ConnectionFactory;
- TDbDriverStatePtr DbDriverState;
+ std::shared_ptr<IWriteSessionConnectionProcessorFactory> ConnectionFactory;
+ TDbDriverStatePtr DbDriverState;
TStringType PrevToken;
bool UpdateTokenInProgress = false;
TInstant LastTokenUpdate = TInstant::Zero();
- std::shared_ptr<TWriteSessionEventsQueue> EventsQueue;
- NGrpc::IQueueClientContextPtr ClientContext; // Common client context.
- NGrpc::IQueueClientContextPtr ConnectContext;
- NGrpc::IQueueClientContextPtr ConnectTimeoutContext;
- NGrpc::IQueueClientContextPtr ConnectDelayContext;
- size_t ConnectionGeneration = 0;
- size_t ConnectionAttemptsDone = 0;
- TAdaptiveLock Lock;
- IProcessor::TPtr Processor;
- IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting).
- std::shared_ptr<TServerMessage> ServerMessage; // Server message to write server response to.
-
- TString SessionId;
+ std::shared_ptr<TWriteSessionEventsQueue> EventsQueue;
+ NGrpc::IQueueClientContextPtr ClientContext; // Common client context.
+ NGrpc::IQueueClientContextPtr ConnectContext;
+ NGrpc::IQueueClientContextPtr ConnectTimeoutContext;
+ NGrpc::IQueueClientContextPtr ConnectDelayContext;
+ size_t ConnectionGeneration = 0;
+ size_t ConnectionAttemptsDone = 0;
+ TAdaptiveLock Lock;
+ IProcessor::TPtr Processor;
+ IRetryState::TPtr RetryState; // Current retry state (if now we are (re)connecting).
+ std::shared_ptr<TServerMessage> ServerMessage; // Server message to write server response to.
+
+ TString SessionId;
IExecutor::TPtr Executor;
IExecutor::TPtr CompressionExecutor;
- size_t MemoryUsage = 0; //!< Estimated amount of memory used
-
- TMessageBatch CurrentBatch;
-
+ size_t MemoryUsage = 0; //!< Estimated amount of memory used
+
+ TMessageBatch CurrentBatch;
+
std::queue<TOriginalMessage> OriginalMessagesToSend;
std::priority_queue<TBlock, std::vector<TBlock>, Greater> PackedMessagesToSend;
- //! Messages that are sent but yet not acknowledged
+ //! Messages that are sent but yet not acknowledged
std::queue<TOriginalMessage> SentOriginalMessages;
std::queue<TBlock> SentPackedMessage;
-
- const size_t MaxBlockSize = std::numeric_limits<size_t>::max();
- const size_t MaxBlockMessageCount = 1; //!< Max message count that can be packed into a single block. In block version 0 is equal to 1 for compatibility
- bool Connected = false;
- bool Started = false;
- TAtomic Aborting = 0;
- bool SessionEstablished = false;
- ui32 PartitionId = 0;
- ui64 LastSeqNo = 0;
- ui64 MinUnsentSeqNo = 0;
- ui64 SeqNoShift = 0;
- TMaybe<bool> AutoSeqNoMode;
+
+ const size_t MaxBlockSize = std::numeric_limits<size_t>::max();
+ const size_t MaxBlockMessageCount = 1; //!< Max message count that can be packed into a single block. In block version 0 is equal to 1 for compatibility
+ bool Connected = false;
+ bool Started = false;
+ TAtomic Aborting = 0;
+ bool SessionEstablished = false;
+ ui32 PartitionId = 0;
+ ui64 LastSeqNo = 0;
+ ui64 MinUnsentSeqNo = 0;
+ ui64 SeqNoShift = 0;
+ TMaybe<bool> AutoSeqNoMode;
bool ValidateSeqNoMode = false;
-
- NThreading::TPromise<ui64> InitSeqNoPromise;
- bool InitSeqNoSetDone = false;
- TInstant SessionStartedTs;
- TInstant LastCountersUpdateTs = TInstant::Zero();
- TInstant LastCountersLogTs;
- TWriterCounters::TPtr Counters;
- TDuration WakeupInterval;
-
-protected:
- ui64 MessagesAcquired = 0;
-};
-
-class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession {
- friend class NTests::TSimpleWriteSessionTestAdapter;
-
-private:
- using TClientMessage = TWriteSession::TClientMessage;
- using TServerMessage = TWriteSession::TServerMessage;
- using IWriteSessionConnectionProcessorFactory = TWriteSession::IWriteSessionConnectionProcessorFactory;
- using IProcessor = TWriteSession::IProcessor;
-
-public:
- TSimpleBlockingWriteSession(
- const TWriteSessionSettings& settings,
+
+ NThreading::TPromise<ui64> InitSeqNoPromise;
+ bool InitSeqNoSetDone = false;
+ TInstant SessionStartedTs;
+ TInstant LastCountersUpdateTs = TInstant::Zero();
+ TInstant LastCountersLogTs;
+ TWriterCounters::TPtr Counters;
+ TDuration WakeupInterval;
+
+protected:
+ ui64 MessagesAcquired = 0;
+};
+
+class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession {
+ friend class NTests::TSimpleWriteSessionTestAdapter;
+
+private:
+ using TClientMessage = TWriteSession::TClientMessage;
+ using TServerMessage = TWriteSession::TServerMessage;
+ using IWriteSessionConnectionProcessorFactory = TWriteSession::IWriteSessionConnectionProcessorFactory;
+ using IProcessor = TWriteSession::IProcessor;
+
+public:
+ TSimpleBlockingWriteSession(
+ const TWriteSessionSettings& settings,
std::shared_ptr<TPersQueueClient::TImpl> client,
- std::shared_ptr<TGRpcConnectionsImpl> connections,
- TDbDriverStatePtr dbDriverState);
-
- bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(),
- const TDuration& blockTimeout = TDuration::Max()) override;
-
- ui64 GetInitSeqNo() override;
-
- bool Close(TDuration closeTimeout = TDuration::Max()) override;
- bool IsAlive() const override;
-
- TWriterCounters::TPtr GetCounters() override;
-
-protected:
- std::shared_ptr<TWriteSession> Writer;
-
-private:
- TMaybe<TContinuationToken> WaitForToken(const TDuration& timeout);
- void HandleAck(TWriteSessionEvent::TAcksEvent&);
- void HandleReady(TWriteSessionEvent::TReadyToAcceptEvent&);
- void HandleClosed(const TSessionClosedEvent&);
-
- TAdaptiveLock Lock;
+ std::shared_ptr<TGRpcConnectionsImpl> connections,
+ TDbDriverStatePtr dbDriverState);
+
+ bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(),
+ const TDuration& blockTimeout = TDuration::Max()) override;
+
+ ui64 GetInitSeqNo() override;
+
+ bool Close(TDuration closeTimeout = TDuration::Max()) override;
+ bool IsAlive() const override;
+
+ TWriterCounters::TPtr GetCounters() override;
+
+protected:
+ std::shared_ptr<TWriteSession> Writer;
+
+private:
+ TMaybe<TContinuationToken> WaitForToken(const TDuration& timeout);
+ void HandleAck(TWriteSessionEvent::TAcksEvent&);
+ void HandleReady(TWriteSessionEvent::TReadyToAcceptEvent&);
+ void HandleClosed(const TSessionClosedEvent&);
+
+ TAdaptiveLock Lock;
std::queue<TContinuationToken> ContinueTokens;
- bool Closed = false;
-};
-
-
-}; // namespace NYdb::NPersQueue
+ bool Closed = false;
+};
+
+
+}; // namespace NYdb::NPersQueue
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h
index 161a4074ee..ee9aa75a0a 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/persqueue.h
@@ -2,9 +2,9 @@
#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h>
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
-#include <library/cpp/monlib/dynamic_counters/counters.h>
-#include <library/cpp/logger/log.h>
-
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+#include <library/cpp/logger/log.h>
+
#include <util/datetime/base.h>
#include <util/generic/hash.h>
#include <util/generic/maybe.h>
@@ -320,22 +320,22 @@ struct TSessionClosedEvent : public TStatus {
TString DebugString() const;
};
-struct TWriteStat : public TThrRefBase {
- TDuration WriteTime;
- TDuration TotalTimeInPartitionQueue;
- TDuration PartitionQuotedTime;
- TDuration TopicQuotedTime;
- using TPtr = TIntrusivePtr<TWriteStat>;
-};
-
-//enum class ECodec {
-// EWC_RAW,
-// EWC_GZIP,
-// EWC_LZOP,
-// EWC_ZSTD
-//};
-
-
+struct TWriteStat : public TThrRefBase {
+ TDuration WriteTime;
+ TDuration TotalTimeInPartitionQueue;
+ TDuration PartitionQuotedTime;
+ TDuration TopicQuotedTime;
+ using TPtr = TIntrusivePtr<TWriteStat>;
+};
+
+//enum class ECodec {
+// EWC_RAW,
+// EWC_GZIP,
+// EWC_LZOP,
+// EWC_ZSTD
+//};
+
+
enum class EClusterDiscoveryMode {
Auto = 0, // enables cluster discovery only for hostname "logbroker.yandex.net" and "logbroker-prestable.yandex.net"
On,
@@ -343,43 +343,43 @@ enum class EClusterDiscoveryMode {
};
class TContinuationToken : public TMoveOnly {
- friend class TWriteSession;
-private:
+ friend class TWriteSession;
+private:
TContinuationToken() = default;
-};
-
-struct TWriterCounters : public TThrRefBase {
- using TSelf = TWriterCounters;
- using TPtr = TIntrusivePtr<TSelf>;
-
+};
+
+struct TWriterCounters : public TThrRefBase {
+ using TSelf = TWriterCounters;
+ using TPtr = TIntrusivePtr<TSelf>;
+
explicit TWriterCounters(const TIntrusivePtr<NMonitoring::TDynamicCounters>& counters);
-
- NMonitoring::TDynamicCounters::TCounterPtr Errors;
- NMonitoring::TDynamicCounters::TCounterPtr CurrentSessionLifetimeMs;
-
- NMonitoring::TDynamicCounters::TCounterPtr BytesWritten;
- NMonitoring::TDynamicCounters::TCounterPtr MessagesWritten;
- NMonitoring::TDynamicCounters::TCounterPtr BytesWrittenCompressed;
-
- NMonitoring::TDynamicCounters::TCounterPtr BytesInflightUncompressed;
- NMonitoring::TDynamicCounters::TCounterPtr BytesInflightCompressed;
- NMonitoring::TDynamicCounters::TCounterPtr BytesInflightTotal;
- NMonitoring::TDynamicCounters::TCounterPtr MessagesInflight;
-
- //! Histograms reporting % usage of memory limit in time.
- //! Provides a histogram looking like: 10% : 100ms, 20%: 300ms, ... 50%: 200ms, ... 100%: 50ms
- //! Which means that < 10% memory usage was observed for 100ms during the period and 50% usage was observed for 200ms
- //! Used to monitor if the writer successfully deals with data flow provided. Larger values in higher buckets
- //! mean that writer is close to overflow (or being overflown) for major periods of time
- //! 3 histograms stand for:
- //! Total memory usage:
- NMonitoring::THistogramPtr TotalBytesInflightUsageByTime;
- //! Memory usage by messages waiting for comression:
- NMonitoring::THistogramPtr UncompressedBytesInflightUsageByTime;
- //! Memory usage by compressed messages pending for write:
- NMonitoring::THistogramPtr CompressedBytesInflightUsageByTime;
-};
-
+
+ NMonitoring::TDynamicCounters::TCounterPtr Errors;
+ NMonitoring::TDynamicCounters::TCounterPtr CurrentSessionLifetimeMs;
+
+ NMonitoring::TDynamicCounters::TCounterPtr BytesWritten;
+ NMonitoring::TDynamicCounters::TCounterPtr MessagesWritten;
+ NMonitoring::TDynamicCounters::TCounterPtr BytesWrittenCompressed;
+
+ NMonitoring::TDynamicCounters::TCounterPtr BytesInflightUncompressed;
+ NMonitoring::TDynamicCounters::TCounterPtr BytesInflightCompressed;
+ NMonitoring::TDynamicCounters::TCounterPtr BytesInflightTotal;
+ NMonitoring::TDynamicCounters::TCounterPtr MessagesInflight;
+
+ //! Histograms reporting % usage of memory limit in time.
+ //! Provides a histogram looking like: 10% : 100ms, 20%: 300ms, ... 50%: 200ms, ... 100%: 50ms
+ //! Which means that < 10% memory usage was observed for 100ms during the period and 50% usage was observed for 200ms
+ //! Used to monitor if the writer successfully deals with data flow provided. Larger values in higher buckets
+ //! mean that writer is close to overflow (or being overflown) for major periods of time
+ //! 3 histograms stand for:
+ //! Total memory usage:
+ NMonitoring::THistogramPtr TotalBytesInflightUsageByTime;
+ //! Memory usage by messages waiting for comression:
+ NMonitoring::THistogramPtr UncompressedBytesInflightUsageByTime;
+ //! Memory usage by compressed messages pending for write:
+ NMonitoring::THistogramPtr CompressedBytesInflightUsageByTime;
+};
+
struct TReaderCounters : public TThrRefBase {
using TSelf = TReaderCounters;
using TPtr = TIntrusivePtr<TSelf>;
@@ -946,15 +946,15 @@ struct IRetryPolicy {
};
class IExecutor : public TThrRefBase {
-public:
+public:
using TPtr = TIntrusivePtr<IExecutor>;
using TFunction = std::function<void()>;
-
+
// Is executor asynchronous.
virtual bool IsAsync() const = 0;
-
+
// Post function to execute.
- virtual void Post(TFunction&& f) = 0;
+ virtual void Post(TFunction&& f) = 0;
// Start method.
// This method is idempotent.
@@ -974,156 +974,156 @@ private:
private:
bool Started = false;
TAdaptiveLock StartLock;
-};
+};
IExecutor::TPtr CreateThreadPoolExecutorAdapter(std::shared_ptr<IThreadPool> threadPool); // Thread pool is expected to have been started.
IExecutor::TPtr CreateThreadPoolExecutor(size_t threads);
-
-//! Events for write session.
-struct TWriteSessionEvent {
-
- //! Event with acknowledge for written messages.
- struct TWriteAck {
- //! Write result.
- enum EEventState {
- EES_WRITTEN, //! Successfully written.
- EES_ALREADY_WRITTEN, //! Skipped on SeqNo deduplication.
- EES_DISCARDED //! In case of destruction of writer or retry policy discarded future retries in this writer.
- };
- //! Details of successfully written message.
- struct TWrittenMessageDetails {
- ui64 Offset;
- ui64 PartitionId;
- };
- //! Same SeqNo as provided on write.
- ui64 SeqNo;
- EEventState State;
- //! Filled only for EES_WRITTEN. Empty for ALREADY and DISCARDED.
- TMaybe<TWrittenMessageDetails> Details;
- //! Write stats from server. See TWriteStat. nullptr for DISCARDED event.
- TWriteStat::TPtr Stat;
-
- };
-
- struct TAcksEvent {
- //! Acks could be batched from several WriteBatch/Write requests.
- //! Acks for messages from one WriteBatch request could be emitted as several TAcksEvents -
- //! they are provided to client as soon as possible.
- TVector<TWriteAck> Acks;
+
+//! Events for write session.
+struct TWriteSessionEvent {
+
+ //! Event with acknowledge for written messages.
+ struct TWriteAck {
+ //! Write result.
+ enum EEventState {
+ EES_WRITTEN, //! Successfully written.
+ EES_ALREADY_WRITTEN, //! Skipped on SeqNo deduplication.
+ EES_DISCARDED //! In case of destruction of writer or retry policy discarded future retries in this writer.
+ };
+ //! Details of successfully written message.
+ struct TWrittenMessageDetails {
+ ui64 Offset;
+ ui64 PartitionId;
+ };
+ //! Same SeqNo as provided on write.
+ ui64 SeqNo;
+ EEventState State;
+ //! Filled only for EES_WRITTEN. Empty for ALREADY and DISCARDED.
+ TMaybe<TWrittenMessageDetails> Details;
+ //! Write stats from server. See TWriteStat. nullptr for DISCARDED event.
+ TWriteStat::TPtr Stat;
+
+ };
+
+ struct TAcksEvent {
+ //! Acks could be batched from several WriteBatch/Write requests.
+ //! Acks for messages from one WriteBatch request could be emitted as several TAcksEvents -
+ //! they are provided to client as soon as possible.
+ TVector<TWriteAck> Acks;
TString DebugString() const;
- };
-
- //! Indicates that a writer is ready to accept new message(s).
+ };
+
+ //! Indicates that a writer is ready to accept new message(s).
//! Continuation token should be kept and then used in write methods.
- struct TReadyToAcceptEvent {
+ struct TReadyToAcceptEvent {
TContinuationToken ContinuationToken;
TString DebugString() const;
- };
-
+ };
+
using TEvent = std::variant<TAcksEvent, TReadyToAcceptEvent, TSessionClosedEvent>;
-};
-
+};
+
//! Event debug string.
TString DebugString(const TWriteSessionEvent::TEvent& event);
-using TSessionClosedHandler = std::function<void(const TSessionClosedEvent&)>;
-
-//! Settings for write session.
-struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> {
- using TSelf = TWriteSessionSettings;
-
- TWriteSessionSettings() = default;
- TWriteSessionSettings(const TWriteSessionSettings&) = default;
- TWriteSessionSettings(TWriteSessionSettings&&) = default;
- TWriteSessionSettings(const TString& path, const TString& messageGroupId) {
- Path(path);
- MessageGroupId(messageGroupId);
- }
-
- TWriteSessionSettings& operator=(const TWriteSessionSettings&) = default;
- TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default;
-
- //! Path of topic to write.
- FLUENT_SETTING(TString, Path);
-
- //! MessageGroupId (aka SourceId) to use.
- FLUENT_SETTING(TString, MessageGroupId);
-
- //! Write to an exact partition group. Generally server assigns partition group automatically.
- //! Using this option is not recommended unless you know for sure why you need it.
- FLUENT_SETTING_OPTIONAL(ui32, PartitionGroupId);
-
+using TSessionClosedHandler = std::function<void(const TSessionClosedEvent&)>;
+
+//! Settings for write session.
+struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> {
+ using TSelf = TWriteSessionSettings;
+
+ TWriteSessionSettings() = default;
+ TWriteSessionSettings(const TWriteSessionSettings&) = default;
+ TWriteSessionSettings(TWriteSessionSettings&&) = default;
+ TWriteSessionSettings(const TString& path, const TString& messageGroupId) {
+ Path(path);
+ MessageGroupId(messageGroupId);
+ }
+
+ TWriteSessionSettings& operator=(const TWriteSessionSettings&) = default;
+ TWriteSessionSettings& operator=(TWriteSessionSettings&&) = default;
+
+ //! Path of topic to write.
+ FLUENT_SETTING(TString, Path);
+
+ //! MessageGroupId (aka SourceId) to use.
+ FLUENT_SETTING(TString, MessageGroupId);
+
+ //! Write to an exact partition group. Generally server assigns partition group automatically.
+ //! Using this option is not recommended unless you know for sure why you need it.
+ FLUENT_SETTING_OPTIONAL(ui32, PartitionGroupId);
+
//! Preferred LB cluster. Used for multi-cluster installation.
//! If specified cluster is unavailable, session will write to other cluster.
FLUENT_SETTING_OPTIONAL(TString, PreferredCluster);
-
+
//! Write to other clusters if there are problems with connection
//! to the first one.
FLUENT_SETTING_DEFAULT(bool, AllowFallbackToOtherClusters, true);
- //! codec and level to use for data compression prior to write.
- FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP);
- FLUENT_SETTING_DEFAULT(i32, CompressionLevel, 4);
-
- //! Writer will not accept new messages if memory usage exceeds this limit.
- //! Memory usage consists of raw data pending compression and compressed messages being sent.
- FLUENT_SETTING_DEFAULT(ui64, MaxMemoryUsage, 20 * 1024 * 1024);
-
- //! Maximum messages accepted by writer but not written (with confirmation from server).
- //! Writer will not accept new messages after reaching the limit.
- FLUENT_SETTING_DEFAULT(ui32, MaxInflightCount, 100000);
-
- //! Retry policy enables automatic retries for non-fatal errors.
+ //! codec and level to use for data compression prior to write.
+ FLUENT_SETTING_DEFAULT(ECodec, Codec, ECodec::GZIP);
+ FLUENT_SETTING_DEFAULT(i32, CompressionLevel, 4);
+
+ //! Writer will not accept new messages if memory usage exceeds this limit.
+ //! Memory usage consists of raw data pending compression and compressed messages being sent.
+ FLUENT_SETTING_DEFAULT(ui64, MaxMemoryUsage, 20 * 1024 * 1024);
+
+ //! Maximum messages accepted by writer but not written (with confirmation from server).
+ //! Writer will not accept new messages after reaching the limit.
+ FLUENT_SETTING_DEFAULT(ui32, MaxInflightCount, 100000);
+
+ //! Retry policy enables automatic retries for non-fatal errors.
//! IRetryPolicy::GetDefaultPolicy() if null (not set).
- FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy);
-
- //! User metadata that may be attached to write session.
+ FLUENT_SETTING(IRetryPolicy::TPtr, RetryPolicy);
+
+ //! User metadata that may be attached to write session.
TWriteSessionSettings& AppendSessionMeta(const TString& key, const TString& value) {
Meta_.Fields[key] = value;
return *this;
};
- NYdb::NPersQueue::TWriteSessionMeta Meta_;
-
- //! Writer will accumulate messages until reaching up to BatchFlushSize bytes
- //! but for no longer than BatchFlushInterval.
- //! Upon reaching FlushInterval or FlushSize limit, all messages will be written with one batch.
- //! Greatly increases performance for small messages.
- //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients
- //! sending small messages at high rate).
- FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval);
- FLUENT_SETTING_OPTIONAL(ui64, BatchFlushSizeBytes);
-
+ NYdb::NPersQueue::TWriteSessionMeta Meta_;
+
+ //! Writer will accumulate messages until reaching up to BatchFlushSize bytes
+ //! but for no longer than BatchFlushInterval.
+ //! Upon reaching FlushInterval or FlushSize limit, all messages will be written with one batch.
+ //! Greatly increases performance for small messages.
+ //! Setting either value to zero means immediate write with no batching. (Unrecommended, especially for clients
+ //! sending small messages at high rate).
+ FLUENT_SETTING_OPTIONAL(TDuration, BatchFlushInterval);
+ FLUENT_SETTING_OPTIONAL(ui64, BatchFlushSizeBytes);
+
FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30));
-
- FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters);
-
+
+ FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters);
+
//! Executor for compression tasks.
//! If not set, default executor will be used.
FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor);
- struct TEventHandlers {
- using TSelf = TEventHandlers;
- using TWriteAckHandler = std::function<void(TWriteSessionEvent::TAcksEvent&)>;
- using TReadyToAcceptHandler = std::function<void(TWriteSessionEvent::TReadyToAcceptEvent&)>;
-
- //! Function to handle Acks events.
- //! If this handler is set, write ack events will be handled by handler,
- //! otherwise sent to TWriteSession::GetEvent().
- FLUENT_SETTING(TWriteAckHandler, AcksHandler);
-
- //! Function to handle ReadyToAccept event.
- //! If this handler is set, write these events will be handled by handler,
- //! otherwise sent to TWriteSession::GetEvent().
- FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHander);
-
- //! Function to handle close session events.
- //! If this handler is set, close session events will be handled by handler
- //! and then sent to TWriteSession::GetEvent().
- FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler);
+ struct TEventHandlers {
+ using TSelf = TEventHandlers;
+ using TWriteAckHandler = std::function<void(TWriteSessionEvent::TAcksEvent&)>;
+ using TReadyToAcceptHandler = std::function<void(TWriteSessionEvent::TReadyToAcceptEvent&)>;
+
+ //! Function to handle Acks events.
+ //! If this handler is set, write ack events will be handled by handler,
+ //! otherwise sent to TWriteSession::GetEvent().
+ FLUENT_SETTING(TWriteAckHandler, AcksHandler);
+
+ //! Function to handle ReadyToAccept event.
+ //! If this handler is set, write these events will be handled by handler,
+ //! otherwise sent to TWriteSession::GetEvent().
+ FLUENT_SETTING(TReadyToAcceptHandler, ReadyToAcceptHander);
+
+ //! Function to handle close session events.
+ //! If this handler is set, close session events will be handled by handler
+ //! and then sent to TWriteSession::GetEvent().
+ FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler);
//! Function to handle all event types.
//! If event with current type has no handler for this type of event,
@@ -1134,10 +1134,10 @@ struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> {
//! Executor for handlers.
//! If not set, default single threaded executor will be used.
FLUENT_SETTING(IExecutor::TPtr, HandlersExecutor);
- };
-
- //! Event handlers.
- FLUENT_SETTING(TEventHandlers, EventHandlers);
+ };
+
+ //! Event handlers.
+ FLUENT_SETTING(TEventHandlers, EventHandlers);
//! Enables validation of SeqNo. If enabled, then writer will check writing with seqNo and without it and throws exception.
FLUENT_SETTING_DEFAULT(bool, ValidateSeqNo, true);
@@ -1145,8 +1145,8 @@ struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> {
//! Manages cluster discovery mode.
FLUENT_SETTING_OPTIONAL(EClusterDiscoveryMode, ClusterDiscoveryMode);
-};
-
+};
+
//! Read settings for single topic.
struct TTopicReadSettings {
using TSelf = TTopicReadSettings;
@@ -1237,7 +1237,7 @@ struct TReadSessionSettings : public TRequestSettings<TReadSessionSettings> {
//! If this handler is set, close session events will be handled by handler
//! and then sent to TReadSession::GetEvent().
//! Default value is empty function (not set).
- FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler);
+ FLUENT_SETTING(TSessionClosedHandler, SessionClosedHandler);
//! Function to handle all event types.
//! If event with current type has no handler for this type of event,
@@ -1329,71 +1329,71 @@ struct TReadSessionSettings : public TRequestSettings<TReadSessionSettings> {
};
//! Simple write session. Does not need event handlers. Does not provide Events, ContinuationTokens, write Acks.
-class ISimpleBlockingWriteSession : public TThrRefBase {
-public:
- //! Write single message. Blocks for up to blockTimeout if inflight is full or memoryUsage is exceeded;
- //! return - true if write succeeded, false if message was not enqueued for write within blockTimeout.
- //! no Ack is provided.
- virtual bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(),
- const TDuration& blockTimeout = TDuration::Max()) = 0;
-
- //! Blocks till SeqNo is discovered from server. Returns 0 in case of failure on init.
- virtual ui64 GetInitSeqNo() = 0;
-
- //! Complete all active writes, wait for ack from server and close.
- //! closeTimeout - max time to wait. Empty Maybe means infinity.
- //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted.
-
- virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0;
-
- //! Returns true if write session is alive and acitve. False if session was closed.
- virtual bool IsAlive() const = 0;
-
- virtual TWriterCounters::TPtr GetCounters() = 0;
-
- //! Close immediately and destroy, don't wait for anything.
- virtual ~ISimpleBlockingWriteSession() = default;
-};
-
-//! Generic write session with all capabilities.
-class IWriteSession {
-public:
- //! Future that is set when next event is available.
- virtual NThreading::TFuture<void> WaitEvent() = 0;
-
+class ISimpleBlockingWriteSession : public TThrRefBase {
+public:
+ //! Write single message. Blocks for up to blockTimeout if inflight is full or memoryUsage is exceeded;
+ //! return - true if write succeeded, false if message was not enqueued for write within blockTimeout.
+ //! no Ack is provided.
+ virtual bool Write(TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing(),
+ const TDuration& blockTimeout = TDuration::Max()) = 0;
+
+ //! Blocks till SeqNo is discovered from server. Returns 0 in case of failure on init.
+ virtual ui64 GetInitSeqNo() = 0;
+
+ //! Complete all active writes, wait for ack from server and close.
+ //! closeTimeout - max time to wait. Empty Maybe means infinity.
+ //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted.
+
+ virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0;
+
+ //! Returns true if write session is alive and acitve. False if session was closed.
+ virtual bool IsAlive() const = 0;
+
+ virtual TWriterCounters::TPtr GetCounters() = 0;
+
+ //! Close immediately and destroy, don't wait for anything.
+ virtual ~ISimpleBlockingWriteSession() = default;
+};
+
+//! Generic write session with all capabilities.
+class IWriteSession {
+public:
+ //! Future that is set when next event is available.
+ virtual NThreading::TFuture<void> WaitEvent() = 0;
+
//! Wait and return next event. Use WaitEvent() for non-blocking wait.
virtual TMaybe<TWriteSessionEvent::TEvent> GetEvent(bool block = false) = 0;
-
- //! Get several events in one call.
- //! If blocking = false, instantly returns up to maxEventsCount available events.
- //! If blocking = true, blocks till maxEventsCount events are available.
- //! If maxEventsCount is unset, write session decides the count to return itself.
+
+ //! Get several events in one call.
+ //! If blocking = false, instantly returns up to maxEventsCount available events.
+ //! If blocking = true, blocks till maxEventsCount events are available.
+ //! If maxEventsCount is unset, write session decides the count to return itself.
virtual TVector<TWriteSessionEvent::TEvent> GetEvents(bool block = false, TMaybe<size_t> maxEventsCount = Nothing()) = 0;
-
- //! Future that is set when initial SeqNo is available.
- virtual NThreading::TFuture<ui64> GetInitSeqNo() = 0;
-
- //! Write single message.
+
+ //! Future that is set when initial SeqNo is available.
+ virtual NThreading::TFuture<ui64> GetInitSeqNo() = 0;
+
+ //! Write single message.
//! continuationToken - a token earlier provided to client with ReadyToAccept event.
- virtual void Write(TContinuationToken&& continuationToken, TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) = 0;
-
+ virtual void Write(TContinuationToken&& continuationToken, TStringBuf data, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) = 0;
+
//! Write single message that is already coded by codec. Codec from settings does not apply to this message.
//! continuationToken - a token earlier provided to client with ReadyToAccept event.
//! originalSize - size of unpacked message
virtual void WriteEncoded(TContinuationToken&& continuationToken, TStringBuf data, ECodec codec, ui32 originalSize, TMaybe<ui64> seqNo = Nothing(), TMaybe<TInstant> createTimestamp = Nothing()) = 0;
- //! Wait for all writes to complete (no more that closeTimeout()), than close. Empty maybe - means infinite timeout.
- //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted.
- virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0;
-
- //! Writer counters with different stats (see TWriterConuters).
- virtual TWriterCounters::TPtr GetCounters() = 0;
-
- //! Close() with timeout = 0 and destroy everything instantly.
- virtual ~IWriteSession() = default;
-};
-
+ //! Wait for all writes to complete (no more that closeTimeout()), than close. Empty maybe - means infinite timeout.
+ //! return - true if all writes were completed and acked. false if timeout was reached and some writes were aborted.
+ virtual bool Close(TDuration closeTimeout = TDuration::Max()) = 0;
+
+ //! Writer counters with different stats (see TWriterConuters).
+ virtual TWriterCounters::TPtr GetCounters() = 0;
+
+ //! Close() with timeout = 0 and destroy everything instantly.
+ virtual ~IWriteSession() = default;
+};
+
class IReadSession {
public:
//! Main reader loop.
@@ -1493,8 +1493,8 @@ public:
std::shared_ptr<IReadSession> CreateReadSession(const TReadSessionSettings& settings);
//! Create write session.
- std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleBlockingWriteSession(const TWriteSessionSettings& settings);
- std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings);
+ std::shared_ptr<ISimpleBlockingWriteSession> CreateSimpleBlockingWriteSession(const TWriteSessionSettings& settings);
+ std::shared_ptr<IWriteSession> CreateWriteSession(const TWriteSessionSettings& settings);
private:
std::shared_ptr<TImpl> Impl_;
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp
index a3505e77fb..7e0ac17404 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/basic_usage_ut.cpp
@@ -1,223 +1,223 @@
-#include "ut_utils.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-#include <library/cpp/threading/future/future.h>
-#include <library/cpp/threading/future/async.h>
+#include "ut_utils.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/threading/future/future.h>
+#include <library/cpp/threading/future/async.h>
#include <library/cpp/threading/chunk_queue/queue.h>
-#include <util/generic/ptr.h>
-#include <util/generic/queue.h>
-#include <util/string/join.h>
-#include <util/system/event.h>
+#include <util/generic/ptr.h>
+#include <util/generic/queue.h>
+#include <util/string/join.h>
+#include <util/system/event.h>
#include <util/stream/zlib.h>
#include <util/stream/str.h>
-#include <atomic>
-
-using namespace NThreading;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-using namespace NPersQueue;
-
-namespace NYdb::NPersQueue::NTests {
-// This suite tests basic Logbroker usage scenario: write and read a bunch of messages
-Y_UNIT_TEST_SUITE(BasicUsage) {
- template<typename TResponse>
- void AssertStreamingMessageCase(const typename TResponse::ResponseCase& expectedCase, const TResponse& actualResponse) {
- UNIT_ASSERT_EQUAL_C(expectedCase, actualResponse.GetResponseCase(), "Got unexpected streaming message or error:\n" + actualResponse.DebugString());
- }
-
- using TWriteCallable = std::function<TFuture<TWriteResult>(TString& data, ui64 sequenceNumber, TInstant createdAt)>;
-
+#include <atomic>
+
+using namespace NThreading;
+using namespace NKikimr;
+using namespace NKikimr::NPersQueueTests;
+using namespace NPersQueue;
+
+namespace NYdb::NPersQueue::NTests {
+// This suite tests basic Logbroker usage scenario: write and read a bunch of messages
+Y_UNIT_TEST_SUITE(BasicUsage) {
+ template<typename TResponse>
+ void AssertStreamingMessageCase(const typename TResponse::ResponseCase& expectedCase, const TResponse& actualResponse) {
+ UNIT_ASSERT_EQUAL_C(expectedCase, actualResponse.GetResponseCase(), "Got unexpected streaming message or error:\n" + actualResponse.DebugString());
+ }
+
+ using TWriteCallable = std::function<TFuture<TWriteResult>(TString& data, ui64 sequenceNumber, TInstant createdAt)>;
+
void WriteAndReadAndCommitRandomMessages(TPersQueueYdbSdkTestSetup* setup, TWriteCallable write, bool disableClusterDiscovery = false) {
- auto log = setup->GetLog();
- const TInstant start = TInstant::Now();
- TVector<TString> messages;
- const ui32 messageCount = 5;
- ui32 totalSize = 0;
- std::queue<NThreading::TFuture<TWriteResult>> writeFutures;
- for (ui32 i = 1; i <= messageCount; ++i) {
- const size_t size = (1 + (i%100))*1_KB;
-
- const auto timestamp = TInstant::MilliSeconds(i);
- const auto message = NUnitTest::RandomString(size, std::rand());
-
- messages.emplace_back(std::move(message));
-
- auto writeFuture = write(messages.back(), i, timestamp);
- writeFutures.push(writeFuture);
- totalSize += size;
- }
-
- while (!writeFutures.empty()) {
- const auto& oldestWriteFuture = writeFutures.front();
- const auto& result = oldestWriteFuture.GetValueSync();
- UNIT_ASSERT_C(result.Ok || result.NoWait, result.ResponseDebugString);
- writeFutures.pop();
- }
-
- log.Write(TLOG_INFO, "All messages are written");
-
- std::shared_ptr<IReadSession> readSession = setup->GetPersQueueClient().CreateReadSession(
+ auto log = setup->GetLog();
+ const TInstant start = TInstant::Now();
+ TVector<TString> messages;
+ const ui32 messageCount = 5;
+ ui32 totalSize = 0;
+ std::queue<NThreading::TFuture<TWriteResult>> writeFutures;
+ for (ui32 i = 1; i <= messageCount; ++i) {
+ const size_t size = (1 + (i%100))*1_KB;
+
+ const auto timestamp = TInstant::MilliSeconds(i);
+ const auto message = NUnitTest::RandomString(size, std::rand());
+
+ messages.emplace_back(std::move(message));
+
+ auto writeFuture = write(messages.back(), i, timestamp);
+ writeFutures.push(writeFuture);
+ totalSize += size;
+ }
+
+ while (!writeFutures.empty()) {
+ const auto& oldestWriteFuture = writeFutures.front();
+ const auto& result = oldestWriteFuture.GetValueSync();
+ UNIT_ASSERT_C(result.Ok || result.NoWait, result.ResponseDebugString);
+ writeFutures.pop();
+ }
+
+ log.Write(TLOG_INFO, "All messages are written");
+
+ std::shared_ptr<IReadSession> readSession = setup->GetPersQueueClient().CreateReadSession(
setup->GetReadSessionSettings().DisableClusterDiscovery(disableClusterDiscovery)
- );
-// auto isStarted = consumer->Start().ExtractValueSync();
-// AssertStreamingMessageCase(TReadResponse::kInit, isStarted.Response);
-
- ui32 readMessageCount = 0;
- TMaybe<ui32> committedOffset;
- ui32 previousOffset = 0;
- bool closed = false;
+ );
+// auto isStarted = consumer->Start().ExtractValueSync();
+// AssertStreamingMessageCase(TReadResponse::kInit, isStarted.Response);
+
+ ui32 readMessageCount = 0;
+ TMaybe<ui32> committedOffset;
+ ui32 previousOffset = 0;
+ bool closed = false;
while ((readMessageCount < messageCount || committedOffset <= previousOffset) && !closed) {
- Cerr << "Get event on client\n";
- auto event = *readSession->GetEvent(true);
- std::visit(TOverloaded {
- [&](TReadSessionEvent::TDataReceivedEvent& event) {
- for (auto& message: event.GetMessages()) {
- TString sourceId = message.GetMessageGroupId();
- ui32 seqNo = message.GetSeqNo();
- UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo);
- ++readMessageCount;
- auto offset = message.GetOffset();
- if (readMessageCount == 1) {
- UNIT_ASSERT_VALUES_EQUAL(offset, 0);
- } else {
- UNIT_ASSERT_VALUES_EQUAL(offset, previousOffset + 1);
- }
- previousOffset = offset;
- message.Commit();
- }
- },
- [&](TReadSessionEvent::TCommitAcknowledgementEvent& event) {
- auto offset = event.GetCommittedOffset();
- log << TLOG_INFO << "Got commit ack with offset " << offset;
- if (committedOffset.Defined()) {
- UNIT_ASSERT_GT(offset, *committedOffset);
- }
- committedOffset = offset;
- },
- [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) {
- //UNIT_FAIL("Test does not support lock sessions yet");
- event.Confirm();
- },
- [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) {
-// UNIT_FAIL("Test does not support lock sessions yet");
- event.Confirm();
- },
- [&](TReadSessionEvent::TPartitionStreamStatusEvent&) {
- UNIT_FAIL("Test does not support lock sessions yet");
- },
- [&](TReadSessionEvent::TPartitionStreamClosedEvent&) {
- UNIT_FAIL("Test does not support lock sessions yet");
- },
- [&](TSessionClosedEvent& event) {
- log << TLOG_INFO << "Got close event: " << event.DebugString();
- //UNIT_FAIL("Session closed");
- closed = true;
- }
-
- }, event);
- log << TLOG_INFO << "Read message count is " << readMessageCount << " (from " << messageCount
- << " in total), committed offset count is "
- << (committedOffset.Defined() ? * committedOffset : 0) << " (maximum offset: "
- << previousOffset << ")";
- }
-
+ Cerr << "Get event on client\n";
+ auto event = *readSession->GetEvent(true);
+ std::visit(TOverloaded {
+ [&](TReadSessionEvent::TDataReceivedEvent& event) {
+ for (auto& message: event.GetMessages()) {
+ TString sourceId = message.GetMessageGroupId();
+ ui32 seqNo = message.GetSeqNo();
+ UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo);
+ ++readMessageCount;
+ auto offset = message.GetOffset();
+ if (readMessageCount == 1) {
+ UNIT_ASSERT_VALUES_EQUAL(offset, 0);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(offset, previousOffset + 1);
+ }
+ previousOffset = offset;
+ message.Commit();
+ }
+ },
+ [&](TReadSessionEvent::TCommitAcknowledgementEvent& event) {
+ auto offset = event.GetCommittedOffset();
+ log << TLOG_INFO << "Got commit ack with offset " << offset;
+ if (committedOffset.Defined()) {
+ UNIT_ASSERT_GT(offset, *committedOffset);
+ }
+ committedOffset = offset;
+ },
+ [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) {
+ //UNIT_FAIL("Test does not support lock sessions yet");
+ event.Confirm();
+ },
+ [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) {
+// UNIT_FAIL("Test does not support lock sessions yet");
+ event.Confirm();
+ },
+ [&](TReadSessionEvent::TPartitionStreamStatusEvent&) {
+ UNIT_FAIL("Test does not support lock sessions yet");
+ },
+ [&](TReadSessionEvent::TPartitionStreamClosedEvent&) {
+ UNIT_FAIL("Test does not support lock sessions yet");
+ },
+ [&](TSessionClosedEvent& event) {
+ log << TLOG_INFO << "Got close event: " << event.DebugString();
+ //UNIT_FAIL("Session closed");
+ closed = true;
+ }
+
+ }, event);
+ log << TLOG_INFO << "Read message count is " << readMessageCount << " (from " << messageCount
+ << " in total), committed offset count is "
+ << (committedOffset.Defined() ? * committedOffset : 0) << " (maximum offset: "
+ << previousOffset << ")";
+ }
+
UNIT_ASSERT_VALUES_EQUAL(previousOffset + 1, committedOffset);
- UNIT_ASSERT_VALUES_EQUAL(readMessageCount, messageCount);
- log.Write(TLOG_INFO, Sprintf("Time took to write and read %u messages, %u [MiB] in total is %lu [s]", messageCount, (totalSize / 1024 / 1024), (TInstant::Now() - start).Seconds()));
- }
-
-
- void SimpleWriteAndValidateData(
- TPersQueueYdbSdkTestSetup* setup, TWriteSessionSettings& writeSettings, ui64 count,
- TMaybe<bool> shouldCaptureData = Nothing()
- ) {
- auto& client = setup->GetPersQueueClient();
- auto session = client.CreateSimpleBlockingWriteSession(writeSettings);
- TString messageBase = "message-";
- TVector<TString> sentMessages;
-
- for (auto i = 0u; i < count; i++) {
- sentMessages.emplace_back(messageBase * (i+1) + ToString(i));
- auto res = session->Write(sentMessages.back());
- UNIT_ASSERT(res);
- }
- {
- auto sessionAdapter = TSimpleWriteSessionTestAdapter(
- dynamic_cast<TSimpleBlockingWriteSession *>(session.get()));
- if (shouldCaptureData.Defined()) {
- TStringBuilder msg;
- msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount()
- << " messages, capturing was expected: " << *shouldCaptureData << Endl;
- UNIT_ASSERT_VALUES_EQUAL_C(sessionAdapter.GetAcquiredMessagesCount() > 0, *shouldCaptureData, msg.c_str());
- }
- }
- session->Close();
-
- auto readSettings = setup->GetReadSessionSettings();
- NThreading::TPromise<void> checkedPromise = NThreading::NewPromise<void>();
- auto totalReceived = 0u;
- readSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) {
- auto& messages = ev.GetMessages();
- for (size_t i = 0u; i < messages.size(); ++i) {
- auto& message = messages[i];
- UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]);
- totalReceived++;
- }
- if (totalReceived == sentMessages.size())
- checkedPromise.SetValue();
- });
- auto readSession = client.CreateReadSession(readSettings);
- checkedPromise.GetFuture().GetValueSync();
- readSession->Close(TDuration::Zero());
- }
-
- Y_UNIT_TEST(WriteAndReadSomeMessagesWithAsyncCompression) {
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- TWriteSessionSettings writeSettings;
- writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id");
- SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true);
- }
-
- Y_UNIT_TEST(WriteAndReadSomeMessagesWithSyncCompression) {
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- TWriteSessionSettings writeSettings;
- writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id");
- IExecutor::TPtr executor = new TSyncExecutor();
- writeSettings.CompressionExecutor(executor);
- // LOGBROKER-7189
- //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false);
- SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true);
- }
-
- Y_UNIT_TEST(WriteAndReadSomeMessagesWithNoCompression) {
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- TWriteSessionSettings writeSettings;
- writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id");
- writeSettings.Codec(ECodec::RAW);
- // LOGBROKER-7189
- //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false);
- SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true);
- }
-
- Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessages) {
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- auto log = setup->GetLog();
- TYDBClientEventLoop clientEventLoop{setup};
-
- TAutoEvent messagesWrittenToBuffer;
- auto clientWrite = [&](TString& message, ui64 sequenceNumber, TInstant createdAt) {
- auto promise = NewPromise<TWriteResult>();
- //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber;
- clientEventLoop.MessageBuffer.Enqueue(TAcknowledgableMessage{message, sequenceNumber, createdAt, promise});
- messagesWrittenToBuffer.Signal();
- return promise.GetFuture();
- };
-
- WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite));
- }
-
+ UNIT_ASSERT_VALUES_EQUAL(readMessageCount, messageCount);
+ log.Write(TLOG_INFO, Sprintf("Time took to write and read %u messages, %u [MiB] in total is %lu [s]", messageCount, (totalSize / 1024 / 1024), (TInstant::Now() - start).Seconds()));
+ }
+
+
+ void SimpleWriteAndValidateData(
+ TPersQueueYdbSdkTestSetup* setup, TWriteSessionSettings& writeSettings, ui64 count,
+ TMaybe<bool> shouldCaptureData = Nothing()
+ ) {
+ auto& client = setup->GetPersQueueClient();
+ auto session = client.CreateSimpleBlockingWriteSession(writeSettings);
+ TString messageBase = "message-";
+ TVector<TString> sentMessages;
+
+ for (auto i = 0u; i < count; i++) {
+ sentMessages.emplace_back(messageBase * (i+1) + ToString(i));
+ auto res = session->Write(sentMessages.back());
+ UNIT_ASSERT(res);
+ }
+ {
+ auto sessionAdapter = TSimpleWriteSessionTestAdapter(
+ dynamic_cast<TSimpleBlockingWriteSession *>(session.get()));
+ if (shouldCaptureData.Defined()) {
+ TStringBuilder msg;
+ msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount()
+ << " messages, capturing was expected: " << *shouldCaptureData << Endl;
+ UNIT_ASSERT_VALUES_EQUAL_C(sessionAdapter.GetAcquiredMessagesCount() > 0, *shouldCaptureData, msg.c_str());
+ }
+ }
+ session->Close();
+
+ auto readSettings = setup->GetReadSessionSettings();
+ NThreading::TPromise<void> checkedPromise = NThreading::NewPromise<void>();
+ auto totalReceived = 0u;
+ readSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) {
+ auto& messages = ev.GetMessages();
+ for (size_t i = 0u; i < messages.size(); ++i) {
+ auto& message = messages[i];
+ UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]);
+ totalReceived++;
+ }
+ if (totalReceived == sentMessages.size())
+ checkedPromise.SetValue();
+ });
+ auto readSession = client.CreateReadSession(readSettings);
+ checkedPromise.GetFuture().GetValueSync();
+ readSession->Close(TDuration::Zero());
+ }
+
+ Y_UNIT_TEST(WriteAndReadSomeMessagesWithAsyncCompression) {
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ TWriteSessionSettings writeSettings;
+ writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id");
+ SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true);
+ }
+
+ Y_UNIT_TEST(WriteAndReadSomeMessagesWithSyncCompression) {
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ TWriteSessionSettings writeSettings;
+ writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id");
+ IExecutor::TPtr executor = new TSyncExecutor();
+ writeSettings.CompressionExecutor(executor);
+ // LOGBROKER-7189
+ //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false);
+ SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true);
+ }
+
+ Y_UNIT_TEST(WriteAndReadSomeMessagesWithNoCompression) {
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ TWriteSessionSettings writeSettings;
+ writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id");
+ writeSettings.Codec(ECodec::RAW);
+ // LOGBROKER-7189
+ //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false);
+ SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true);
+ }
+
+ Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessages) {
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ auto log = setup->GetLog();
+ TYDBClientEventLoop clientEventLoop{setup};
+
+ TAutoEvent messagesWrittenToBuffer;
+ auto clientWrite = [&](TString& message, ui64 sequenceNumber, TInstant createdAt) {
+ auto promise = NewPromise<TWriteResult>();
+ //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber;
+ clientEventLoop.MessageBuffer.Enqueue(TAcknowledgableMessage{message, sequenceNumber, createdAt, promise});
+ messagesWrittenToBuffer.Signal();
+ return promise.GetFuture();
+ };
+
+ WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite));
+ }
+
Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessagesNoClusterDiscovery) {
auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
auto log = setup->GetLog();
@@ -235,49 +235,49 @@ Y_UNIT_TEST_SUITE(BasicUsage) {
WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite), true);
}
- Y_UNIT_TEST(TSimpleWriteSession_AutoSeqNo_BasicUsage) {
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- auto& client = setup->GetPersQueueClient();
- auto settings = setup->GetWriteSessionSettings();
- auto simpleSession = client.CreateSimpleBlockingWriteSession(settings);
-
- TString msg = "message";
- auto clientWrite = [&](TString& message, ui64 seqNo, TInstant createdAt) {
- auto promise = NewPromise<TWriteResult>();
- //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber;
- simpleSession->Write(message, seqNo, createdAt);
- promise.SetValue(TWriteResult{true});
- return promise.GetFuture();
- };
- WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite));
- auto res = simpleSession->Close();
- UNIT_ASSERT(res);
- UNIT_ASSERT(res);
- }
-
-
-
- Y_UNIT_TEST(TWriteSession_AutoBatching) {
- // ToDo: Re-enable once batching takes more than 1 message at once
- return;
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- auto& client = setup->GetPersQueueClient();
- auto settings = setup->GetWriteSessionSettings();
- size_t batchSize = 100;
- settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time.
- settings.BatchFlushSizeBytes(batchSize);
- auto writer = client.CreateWriteSession(settings);
- TString message = "message";
- size_t totalSize = 0, seqNo = 0;
- while (totalSize < batchSize) {
- auto event = *writer->GetEvent(true);
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
- auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
- writer->Write(std::move(continueToken), message, ++seqNo);
- totalSize += message.size();
- }
- WaitMessagesAcked(writer, 1, seqNo);
- }
+ Y_UNIT_TEST(TSimpleWriteSession_AutoSeqNo_BasicUsage) {
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ auto& client = setup->GetPersQueueClient();
+ auto settings = setup->GetWriteSessionSettings();
+ auto simpleSession = client.CreateSimpleBlockingWriteSession(settings);
+
+ TString msg = "message";
+ auto clientWrite = [&](TString& message, ui64 seqNo, TInstant createdAt) {
+ auto promise = NewPromise<TWriteResult>();
+ //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber;
+ simpleSession->Write(message, seqNo, createdAt);
+ promise.SetValue(TWriteResult{true});
+ return promise.GetFuture();
+ };
+ WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite));
+ auto res = simpleSession->Close();
+ UNIT_ASSERT(res);
+ UNIT_ASSERT(res);
+ }
+
+
+
+ Y_UNIT_TEST(TWriteSession_AutoBatching) {
+ // ToDo: Re-enable once batching takes more than 1 message at once
+ return;
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ auto& client = setup->GetPersQueueClient();
+ auto settings = setup->GetWriteSessionSettings();
+ size_t batchSize = 100;
+ settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time.
+ settings.BatchFlushSizeBytes(batchSize);
+ auto writer = client.CreateWriteSession(settings);
+ TString message = "message";
+ size_t totalSize = 0, seqNo = 0;
+ while (totalSize < batchSize) {
+ auto event = *writer->GetEvent(true);
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
+ auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
+ writer->Write(std::move(continueToken), message, ++seqNo);
+ totalSize += message.size();
+ }
+ WaitMessagesAcked(writer, 1, seqNo);
+ }
Y_UNIT_TEST(TWriteSession_WriteEncoded) {
auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
@@ -379,36 +379,36 @@ Y_UNIT_TEST_SUITE(BasicUsage) {
}
- Y_UNIT_TEST(TWriteSession_BatchingProducesContinueTokens) {
- // ToDo: Re-enable once batching takes more than 1 message at once
- return;
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- auto& client = setup->GetPersQueueClient();
- auto settings = setup->GetWriteSessionSettings();
- auto batchInterval = TDuration::Seconds(10);
- settings.BatchFlushInterval(batchInterval); // Batch on size, not on time.
- settings.BatchFlushSizeBytes(9999999999);
- settings.MaxMemoryUsage(99);
- auto writer = client.CreateWriteSession(settings);
- TString message = "0123456789";
- auto seqNo = 10u;
-
- auto event = *writer->GetEvent(true);
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
- auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
- writer->Write(std::move(continueToken), message * 5, ++seqNo);
- auto eventF = writer->WaitEvent();
- eventF.Wait(batchInterval / 2);
- UNIT_ASSERT(!eventF.HasValue());
- while (true) {
- event = *writer->GetEvent(true);
- if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) {
- break;
- } else {
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event));
- }
- }
- writer->Close();
- }
-}
-} // namespace NYdb::NPersQueue::NTests
+ Y_UNIT_TEST(TWriteSession_BatchingProducesContinueTokens) {
+ // ToDo: Re-enable once batching takes more than 1 message at once
+ return;
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ auto& client = setup->GetPersQueueClient();
+ auto settings = setup->GetWriteSessionSettings();
+ auto batchInterval = TDuration::Seconds(10);
+ settings.BatchFlushInterval(batchInterval); // Batch on size, not on time.
+ settings.BatchFlushSizeBytes(9999999999);
+ settings.MaxMemoryUsage(99);
+ auto writer = client.CreateWriteSession(settings);
+ TString message = "0123456789";
+ auto seqNo = 10u;
+
+ auto event = *writer->GetEvent(true);
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
+ auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
+ writer->Write(std::move(continueToken), message * 5, ++seqNo);
+ auto eventF = writer->WaitEvent();
+ eventF.Wait(batchInterval / 2);
+ UNIT_ASSERT(!eventF.HasValue());
+ while (true) {
+ event = *writer->GetEvent(true);
+ if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) {
+ break;
+ } else {
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event));
+ }
+ }
+ writer->Close();
+ }
+}
+} // namespace NYdb::NPersQueue::NTests
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp
index 47c6a96562..04abfab569 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/compress_executor_ut.cpp
@@ -1,95 +1,95 @@
#include "ut_utils.h"
-
-namespace NYdb::NPersQueue::NTests {
-
-Y_UNIT_TEST_SUITE(CompressExecutor) {
- Y_UNIT_TEST(TestReorderedExecutor) {
- auto queue = std::make_shared<TLockFreeQueue<ui64>>();
- TYdbPqWriterTestHelper helper(TEST_CASE_NAME, queue);
-
- auto f1 = helper.Write();
- auto f2 = helper.Write();
- f1.Wait(TDuration::Seconds(1));
- UNIT_ASSERT(!f1.HasValue());
- UNIT_ASSERT(!f2.HasValue());
-
- queue->Enqueue(2);
- f2.Wait(TDuration::Seconds(1));
- UNIT_ASSERT(!f1.HasValue());
- UNIT_ASSERT(!f2.HasValue());
- queue->Enqueue(1);
- Cerr << "Waiting for writes to complete.\n";
- f1.Wait();
- f2.Wait();
-
- }
- Y_UNIT_TEST(TestExecutorMemUsage) {
- auto queue = std::make_shared<TLockFreeQueue<ui64>>();
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- auto executor = MakeIntrusive<TYdbPqTestExecutor>(queue);
- auto config = setup->GetWriteSessionSettings();
- auto memUsageLimit = 1_KB;
- config.MaxMemoryUsage(memUsageLimit);
- config.CompressionExecutor(executor);
- auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>();
-
- config.RetryPolicy(retryPolicy);
-
- auto writer = setup->GetPersQueueClient().CreateWriteSession(config);
-
- TMaybe<TContinuationToken> continueToken;
- auto event = *writer->GetEvent(true);
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
- continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
-
- auto waitEventFuture = writer->WaitEvent();
- UNIT_ASSERT(!waitEventFuture.HasValue());
-
- TStringBuilder msgBuilder;
- while (msgBuilder.size() < 100 * 1024) {
- msgBuilder << "0123456789abcdefghijk";
- }
- const ui64 COMPRESSED_SIZE = 305;
- TString message = TString(msgBuilder);
- ui64 seqNo = 1;
- auto doWrite = [&] {
- writer->Write(std::move(*continueToken), message, seqNo);
- ++seqNo;
- };
- doWrite();
- waitEventFuture = writer->WaitEvent();
- waitEventFuture.Wait(TDuration::Seconds(3));
- UNIT_ASSERT(!waitEventFuture.HasValue());
- queue->Enqueue(1);
- event = *writer->GetEvent(true);
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
- continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
- event = *writer->GetEvent(true);
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event));
-
- Cerr << "===Will now kick tablets\n";
- setup->KickTablets();
- retryPolicy->ExpectBreakDown();
- retryPolicy->WaitForRetriesSync(1);
- ui64 currentUsageEstimate = 0;
- while (currentUsageEstimate <= memUsageLimit) {
- doWrite();
- queue->Enqueue(seqNo - 1);
- currentUsageEstimate += COMPRESSED_SIZE;
- if (currentUsageEstimate <= memUsageLimit) {
- event = *writer->GetEvent(true);
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
- continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
- }
- }
- Cerr << "============ UT - Wait event...\n";
- waitEventFuture = writer->WaitEvent();
- waitEventFuture.Wait(TDuration::Seconds(3));
- UNIT_ASSERT(!waitEventFuture.HasValue());
-
- writer = nullptr;
- retryPolicy = nullptr;
- }
-}
-};
-
+
+namespace NYdb::NPersQueue::NTests {
+
+Y_UNIT_TEST_SUITE(CompressExecutor) {
+ Y_UNIT_TEST(TestReorderedExecutor) {
+ auto queue = std::make_shared<TLockFreeQueue<ui64>>();
+ TYdbPqWriterTestHelper helper(TEST_CASE_NAME, queue);
+
+ auto f1 = helper.Write();
+ auto f2 = helper.Write();
+ f1.Wait(TDuration::Seconds(1));
+ UNIT_ASSERT(!f1.HasValue());
+ UNIT_ASSERT(!f2.HasValue());
+
+ queue->Enqueue(2);
+ f2.Wait(TDuration::Seconds(1));
+ UNIT_ASSERT(!f1.HasValue());
+ UNIT_ASSERT(!f2.HasValue());
+ queue->Enqueue(1);
+ Cerr << "Waiting for writes to complete.\n";
+ f1.Wait();
+ f2.Wait();
+
+ }
+ Y_UNIT_TEST(TestExecutorMemUsage) {
+ auto queue = std::make_shared<TLockFreeQueue<ui64>>();
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ auto executor = MakeIntrusive<TYdbPqTestExecutor>(queue);
+ auto config = setup->GetWriteSessionSettings();
+ auto memUsageLimit = 1_KB;
+ config.MaxMemoryUsage(memUsageLimit);
+ config.CompressionExecutor(executor);
+ auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>();
+
+ config.RetryPolicy(retryPolicy);
+
+ auto writer = setup->GetPersQueueClient().CreateWriteSession(config);
+
+ TMaybe<TContinuationToken> continueToken;
+ auto event = *writer->GetEvent(true);
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
+ continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
+
+ auto waitEventFuture = writer->WaitEvent();
+ UNIT_ASSERT(!waitEventFuture.HasValue());
+
+ TStringBuilder msgBuilder;
+ while (msgBuilder.size() < 100 * 1024) {
+ msgBuilder << "0123456789abcdefghijk";
+ }
+ const ui64 COMPRESSED_SIZE = 305;
+ TString message = TString(msgBuilder);
+ ui64 seqNo = 1;
+ auto doWrite = [&] {
+ writer->Write(std::move(*continueToken), message, seqNo);
+ ++seqNo;
+ };
+ doWrite();
+ waitEventFuture = writer->WaitEvent();
+ waitEventFuture.Wait(TDuration::Seconds(3));
+ UNIT_ASSERT(!waitEventFuture.HasValue());
+ queue->Enqueue(1);
+ event = *writer->GetEvent(true);
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
+ continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
+ event = *writer->GetEvent(true);
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event));
+
+ Cerr << "===Will now kick tablets\n";
+ setup->KickTablets();
+ retryPolicy->ExpectBreakDown();
+ retryPolicy->WaitForRetriesSync(1);
+ ui64 currentUsageEstimate = 0;
+ while (currentUsageEstimate <= memUsageLimit) {
+ doWrite();
+ queue->Enqueue(seqNo - 1);
+ currentUsageEstimate += COMPRESSED_SIZE;
+ if (currentUsageEstimate <= memUsageLimit) {
+ event = *writer->GetEvent(true);
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
+ continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
+ }
+ }
+ Cerr << "============ UT - Wait event...\n";
+ waitEventFuture = writer->WaitEvent();
+ waitEventFuture.Wait(TDuration::Seconds(3));
+ UNIT_ASSERT(!waitEventFuture.HasValue());
+
+ writer = nullptr;
+ retryPolicy = nullptr;
+ }
+}
+};
+
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp
index 34b64e2aab..9b2367e3f3 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/read_session_ut.cpp
@@ -665,10 +665,10 @@ void TReadSessionImplTestSetup::AssertNoEvents() {
UNIT_ASSERT(!event);
}
-using NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup;
+using NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup;
Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) {
void ReadSessionImpl(bool close, bool commit, bool explicitlySpecifiedPartitions = false) {
- NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup setup("ReadSession");
+ NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup setup("ReadSession");
setup.WriteToTopic({"message1", "message2"});
auto settings = setup.GetReadSessionSettings();
if (explicitlySpecifiedPartitions) {
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp
index 7070d9e5be..50d050e973 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/retry_policy_ut.cpp
@@ -1,22 +1,22 @@
#include "ut_utils.h"
-#include <library/cpp/threading/future/future.h>
-#include <library/cpp/testing/unittest/registar.h>
-
-
-using namespace NThreading;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-using namespace NPersQueue;
-
-namespace NYdb::NPersQueue::NTests {
-
-Y_UNIT_TEST_SUITE(RetryPolicy) {
- Y_UNIT_TEST(TWriteSession_TestPolicy) {
- TYdbPqWriterTestHelper helper(TEST_CASE_NAME);
- helper.Write(true);
- helper.Policy->Initialized(); // Thus ignoring possible early retries on "cluster initializing"
- auto doBreakDown = [&] () {
+#include <library/cpp/threading/future/future.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+
+using namespace NThreading;
+using namespace NKikimr;
+using namespace NKikimr::NPersQueueTests;
+using namespace NPersQueue;
+
+namespace NYdb::NPersQueue::NTests {
+
+Y_UNIT_TEST_SUITE(RetryPolicy) {
+ Y_UNIT_TEST(TWriteSession_TestPolicy) {
+ TYdbPqWriterTestHelper helper(TEST_CASE_NAME);
+ helper.Write(true);
+ helper.Policy->Initialized(); // Thus ignoring possible early retries on "cluster initializing"
+ auto doBreakDown = [&] () {
helper.Policy->ExpectBreakDown();
NThreading::TPromise<void> retriesPromise = NThreading::NewPromise();
Cerr << "WAIT for retries...\n";
@@ -24,16 +24,16 @@ Y_UNIT_TEST_SUITE(RetryPolicy) {
Cerr << "KICK tablets\n";
helper.Setup->KickTablets();
- auto f1 = helper.Write(false);
- auto f2 = helper.Write();
+ auto f1 = helper.Write(false);
+ auto f2 = helper.Write();
- auto retriesFuture = retriesPromise.GetFuture();
- retriesFuture.Wait();
+ auto retriesFuture = retriesPromise.GetFuture();
+ retriesFuture.Wait();
Cerr << "WAIT for retries done\n";
- NThreading::TPromise<void> repairPromise = NThreading::NewPromise();
- auto repairFuture = repairPromise.GetFuture();
- helper.Policy->WaitForRepair(repairPromise);
+ NThreading::TPromise<void> repairPromise = NThreading::NewPromise();
+ auto repairFuture = repairPromise.GetFuture();
+ helper.Policy->WaitForRepair(repairPromise);
Cerr << "ALLOW tablets\n";
@@ -42,85 +42,85 @@ Y_UNIT_TEST_SUITE(RetryPolicy) {
Cerr << "WAIT for repair\n";
repairFuture.Wait();
Cerr << "REPAIR done\n";
- f1.Wait();
- f2.Wait();
- helper.Write(true);
- };
- doBreakDown();
- doBreakDown();
-
- }
- Y_UNIT_TEST(TWriteSession_TestBrokenPolicy) {
- TYdbPqWriterTestHelper helper(TEST_CASE_NAME);
- helper.Write();
- helper.Policy->Initialized();
- helper.Policy->ExpectFatalBreakDown();
- helper.EventLoop->AllowStop();
- auto f1 = helper.Write(false);
- helper.Setup->KickTablets();
- helper.Write(false);
-
- helper.EventLoop->WaitForStop();
- UNIT_ASSERT(!f1.HasValue());
- helper.Setup = nullptr;
-
- };
-
+ f1.Wait();
+ f2.Wait();
+ helper.Write(true);
+ };
+ doBreakDown();
+ doBreakDown();
+
+ }
+ Y_UNIT_TEST(TWriteSession_TestBrokenPolicy) {
+ TYdbPqWriterTestHelper helper(TEST_CASE_NAME);
+ helper.Write();
+ helper.Policy->Initialized();
+ helper.Policy->ExpectFatalBreakDown();
+ helper.EventLoop->AllowStop();
+ auto f1 = helper.Write(false);
+ helper.Setup->KickTablets();
+ helper.Write(false);
+
+ helper.EventLoop->WaitForStop();
+ UNIT_ASSERT(!f1.HasValue());
+ helper.Setup = nullptr;
+
+ };
+
Y_UNIT_TEST(TWriteSession_RetryOnTargetCluster) {
- auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false);
+ auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false);
SDKTestSetup setup2("RetryOnTargetCluster_Dc2");
- setup1->AddDataCenter("dc2", setup2, false);
- setup1->Start();
- auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>();
- auto settings = setup1->GetWriteSessionSettings();
+ setup1->AddDataCenter("dc2", setup2, false);
+ setup1->Start();
+ auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>();
+ auto settings = setup1->GetWriteSessionSettings();
settings.PreferredCluster("dc2");
settings.AllowFallbackToOtherClusters(false);
- settings.RetryPolicy(retryPolicy);
-
- retryPolicy->Initialized();
- retryPolicy->ExpectBreakDown();
-
- auto& client = setup1->GetPersQueueClient();
- Cerr << "=== Create write session \n";
- auto writer = client.CreateWriteSession(settings);
-
- NThreading::TPromise<void> retriesPromise = NThreading::NewPromise();
- auto retriesFuture = retriesPromise.GetFuture();
- retryPolicy->WaitForRetries(3, retriesPromise);
- Cerr << "=== Wait retries\n";
- retriesFuture.Wait();
-
- Cerr << "=== Enable dc2\n";
- setup1->EnableDataCenter("dc2");
-
- NThreading::TPromise<void> repairPromise = NThreading::NewPromise();
- auto repairFuture = repairPromise.GetFuture();
- retryPolicy->WaitForRepair(repairPromise);
- Cerr << "=== Wait for repair\n";
- repairFuture.Wait();
- Cerr << "=== Close writer\n";
- writer->Close();
- }
-
- Y_UNIT_TEST(TWriteSession_SwitchBackToLocalCluster) {
- Cerr << "====Start test\n";
-
- auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false);
- SDKTestSetup setup2("SwitchBackToLocalCluster", false);
- setup2.SetSingleDataCenter("dc2");
- setup2.AddDataCenter("dc1", *setup1, true);
- setup1->AddDataCenter("dc2", setup2, true);
- setup1->Start();
+ settings.RetryPolicy(retryPolicy);
+
+ retryPolicy->Initialized();
+ retryPolicy->ExpectBreakDown();
+
+ auto& client = setup1->GetPersQueueClient();
+ Cerr << "=== Create write session \n";
+ auto writer = client.CreateWriteSession(settings);
+
+ NThreading::TPromise<void> retriesPromise = NThreading::NewPromise();
+ auto retriesFuture = retriesPromise.GetFuture();
+ retryPolicy->WaitForRetries(3, retriesPromise);
+ Cerr << "=== Wait retries\n";
+ retriesFuture.Wait();
+
+ Cerr << "=== Enable dc2\n";
+ setup1->EnableDataCenter("dc2");
+
+ NThreading::TPromise<void> repairPromise = NThreading::NewPromise();
+ auto repairFuture = repairPromise.GetFuture();
+ retryPolicy->WaitForRepair(repairPromise);
+ Cerr << "=== Wait for repair\n";
+ repairFuture.Wait();
+ Cerr << "=== Close writer\n";
+ writer->Close();
+ }
+
+ Y_UNIT_TEST(TWriteSession_SwitchBackToLocalCluster) {
+ Cerr << "====Start test\n";
+
+ auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false);
+ SDKTestSetup setup2("SwitchBackToLocalCluster", false);
+ setup2.SetSingleDataCenter("dc2");
+ setup2.AddDataCenter("dc1", *setup1, true);
+ setup1->AddDataCenter("dc2", setup2, true);
+ setup1->Start();
setup2.Start(false);
- Cerr << "=== Start session 1\n";
- auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1);
- helper->Write(true);
- auto retryPolicy = helper->Policy;
- retryPolicy->Initialized();
-
- auto waitForReconnect = [&](bool enable) {
- Cerr << "=== Expect breakdown\n";
- retryPolicy->ExpectBreakDown();
+ Cerr << "=== Start session 1\n";
+ auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1);
+ helper->Write(true);
+ auto retryPolicy = helper->Policy;
+ retryPolicy->Initialized();
+
+ auto waitForReconnect = [&](bool enable) {
+ Cerr << "=== Expect breakdown\n";
+ retryPolicy->ExpectBreakDown();
NThreading::TPromise<void> retriesPromise = NThreading::NewPromise();
auto retriesFuture = retriesPromise.GetFuture();
@@ -130,246 +130,246 @@ Y_UNIT_TEST_SUITE(RetryPolicy) {
auto repairFuture = repairPromise.GetFuture();
retryPolicy->WaitForRepair(repairPromise);
- if (enable) {
- Cerr << "===Enabled DC1\n";
- setup1->EnableDataCenter("dc1");
- setup2.EnableDataCenter("dc1");
- } else {
- Cerr << "===Disabled DC1\n";
- setup1->DisableDataCenter("dc1");
- setup2.DisableDataCenter("dc1");
- }
+ if (enable) {
+ Cerr << "===Enabled DC1\n";
+ setup1->EnableDataCenter("dc1");
+ setup2.EnableDataCenter("dc1");
+ } else {
+ Cerr << "===Disabled DC1\n";
+ setup1->DisableDataCenter("dc1");
+ setup2.DisableDataCenter("dc1");
+ }
Sleep(TDuration::Seconds(5));
retriesFuture.Wait();
repairFuture.Wait();
- };
- Cerr << "===Wait for 1st reconnect\n";
- waitForReconnect(false);
- Cerr << "===Wait for 2nd reconnect\n";
- waitForReconnect(true);
- }
-
- Y_UNIT_TEST(TWriteSession_SeqNoShift) {
- auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false);
- SDKTestSetup setup2("SeqNoShift_Dc2", false);
- setup2.SetSingleDataCenter("dc2");
+ };
+ Cerr << "===Wait for 1st reconnect\n";
+ waitForReconnect(false);
+ Cerr << "===Wait for 2nd reconnect\n";
+ waitForReconnect(true);
+ }
+
+ Y_UNIT_TEST(TWriteSession_SeqNoShift) {
+ auto setup1 = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME, false);
+ SDKTestSetup setup2("SeqNoShift_Dc2", false);
+ setup2.SetSingleDataCenter("dc2");
setup2.AddDataCenter("dc1", *setup1, true);
- setup2.Start();
- setup1->AddDataCenter("dc2", setup2, true);
- setup1->Start();
- TString tmpSrcId = "tmp-src-seqno-shift", noShiftSrcId = "tmp-src-no-seqno-shift";
- auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1);
- auto helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId);
- //auto helperNoShift = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId);
-
- auto settings = setup1->GetWriteSessionSettings();
- auto& client = setup1->GetPersQueueClient();
- //! Fill data in dc1 1 with SeqNo = 1..10 for 2 different SrcId
- for (auto i = 0; i != 10; i++) {
- helper->Write(true); // 1
- helper2->Write(true); // 1
- }
- Cerr << "===Disable dc1\n";
- //! Leave only dc2 available
- setup1->DisableDataCenter("dc1");
- helper->Policy->ExpectBreakDown();
- helper->Policy->WaitForRetriesSync(1);
-
- //! Re-create helpers, kill previous sessions. New sessions will connect to dc2.
- helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1);
- helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1, tmpSrcId);
-
- //! Write some data and await confirmation - just to ensure sessions are started.
- helper->Write(true);
- helper2->Write(true);
-
- helper->Policy->ExpectBreakDown();
- Cerr << "===Disable dc2\n";
- //! Leave no available DCs
- setup1->DisableDataCenter("dc2");
- Cerr << "=== Wait for retries after initial dc2 shutdown\n";
- helper->Policy->WaitForRetriesSync(1);
-
- //! Put some data inflight. It cannot be written now, but SeqNo will be assigned.
- for (auto i = 0; i != 4; i++) {
- helper->Write(false);
- helper2->Write(false);
- }
- auto f = helper->Write(false);
- auto f2 = helper2->Write(false);
- //! Enable DC1. Now writers gonna write collected data to DC1 having LastSeqNo = 10
- //! (because of data written and the very beginning), and inflight data has SeqNo assingned = 2..5,
- //! so the SeqNo shift takes place.
- setup1->EnableDataCenter("dc1");
-
- Cerr << "=====Wait for writes to complete\n";
- f.Wait();
- f2.Wait();
- //! Writer1 is not used any more.
- helper->EventLoop->AllowStop();
- helper = nullptr;
-
- helper2->Policy->Initialized();
- helper2->Policy->ExpectBreakDown();
- //! For the second writer, do switchback to dc2.
- setup1->DisableDataCenter("dc1");
- helper2->Policy->WaitForRetriesSync(1);
- //! Put some data inflight again;
- for (auto i = 0; i != 4; i++) {
- helper2->Write(false);
- }
- f = helper2->Write(false);
- Cerr << "===Enable dc2\n";
- setup1->EnableDataCenter("dc2");
- f.Wait();
- helper2->EventLoop->AllowStop();
- helper2->Policy->ExpectBreakDown();
-
- Cerr << "===Enable dc1\n";
- setup1->EnableDataCenter("dc1");
- auto CheckSeqNo = [&] (const TString& dcName, ui64 expectedSeqNo) {
+ setup2.Start();
+ setup1->AddDataCenter("dc2", setup2, true);
+ setup1->Start();
+ TString tmpSrcId = "tmp-src-seqno-shift", noShiftSrcId = "tmp-src-no-seqno-shift";
+ auto helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1);
+ auto helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId);
+ //auto helperNoShift = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, "dc1", setup1, tmpSrcId);
+
+ auto settings = setup1->GetWriteSessionSettings();
+ auto& client = setup1->GetPersQueueClient();
+ //! Fill data in dc1 1 with SeqNo = 1..10 for 2 different SrcId
+ for (auto i = 0; i != 10; i++) {
+ helper->Write(true); // 1
+ helper2->Write(true); // 1
+ }
+ Cerr << "===Disable dc1\n";
+ //! Leave only dc2 available
+ setup1->DisableDataCenter("dc1");
+ helper->Policy->ExpectBreakDown();
+ helper->Policy->WaitForRetriesSync(1);
+
+ //! Re-create helpers, kill previous sessions. New sessions will connect to dc2.
+ helper = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1);
+ helper2 = MakeHolder<TYdbPqWriterTestHelper>("", nullptr, TString(), setup1, tmpSrcId);
+
+ //! Write some data and await confirmation - just to ensure sessions are started.
+ helper->Write(true);
+ helper2->Write(true);
+
+ helper->Policy->ExpectBreakDown();
+ Cerr << "===Disable dc2\n";
+ //! Leave no available DCs
+ setup1->DisableDataCenter("dc2");
+ Cerr << "=== Wait for retries after initial dc2 shutdown\n";
+ helper->Policy->WaitForRetriesSync(1);
+
+ //! Put some data inflight. It cannot be written now, but SeqNo will be assigned.
+ for (auto i = 0; i != 4; i++) {
+ helper->Write(false);
+ helper2->Write(false);
+ }
+ auto f = helper->Write(false);
+ auto f2 = helper2->Write(false);
+ //! Enable DC1. Now writers gonna write collected data to DC1 having LastSeqNo = 10
+ //! (because of data written and the very beginning), and inflight data has SeqNo assingned = 2..5,
+ //! so the SeqNo shift takes place.
+ setup1->EnableDataCenter("dc1");
+
+ Cerr << "=====Wait for writes to complete\n";
+ f.Wait();
+ f2.Wait();
+ //! Writer1 is not used any more.
+ helper->EventLoop->AllowStop();
+ helper = nullptr;
+
+ helper2->Policy->Initialized();
+ helper2->Policy->ExpectBreakDown();
+ //! For the second writer, do switchback to dc2.
+ setup1->DisableDataCenter("dc1");
+ helper2->Policy->WaitForRetriesSync(1);
+ //! Put some data inflight again;
+ for (auto i = 0; i != 4; i++) {
+ helper2->Write(false);
+ }
+ f = helper2->Write(false);
+ Cerr << "===Enable dc2\n";
+ setup1->EnableDataCenter("dc2");
+ f.Wait();
+ helper2->EventLoop->AllowStop();
+ helper2->Policy->ExpectBreakDown();
+
+ Cerr << "===Enable dc1\n";
+ setup1->EnableDataCenter("dc1");
+ auto CheckSeqNo = [&] (const TString& dcName, ui64 expectedSeqNo) {
settings.PreferredCluster(dcName);
settings.AllowFallbackToOtherClusters(false);
settings.RetryPolicy(nullptr); //switch to default policy;
- auto writer = client.CreateWriteSession(settings);
- auto seqNo = writer->GetInitSeqNo().GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL(seqNo, expectedSeqNo);
- writer->Close(TDuration::Zero());
- };
-
- //!check SeqNo in both DC. For writer1 We expect 14 messages in DC1
- //! (10 written initially + 4 wriiten after recoonect) and 1 message in DC2 (only initial message).
- Cerr << "===Check SeqNo writer1, dc2\n";
- CheckSeqNo("dc2", 1);
- Cerr << "===Check SeqNo writer1, dc1\n";
- CheckSeqNo("dc1", 14);
-
- //! Check SeqNo for writer 2; Expect to have 6 messages on DC2 with MaxSeqNo = 6;
- helper2 = nullptr;
- settings.MessageGroupId(tmpSrcId);
- Cerr << "===Check SeqNo writer2 dc2\n";
- //!DC2 has no a ap in SeqNo since 5 messages were written ot dc 1.
- CheckSeqNo("dc2", 11);
- Cerr << "===Check SeqNo writer2 dc1\n";
- CheckSeqNo("dc1", 14);
-
-
- auto readSession = client.CreateReadSession(setup1->GetReadSessionSettings());
-
- bool stop = false;
- THashMap<TString, ui64> seqNoByClusterSrc1 = {
- {"dc1", 0},
- {"dc2", 0}
- };
- auto SeqNoByClusterSrc2 = seqNoByClusterSrc1;
-
- THashMap<TString, ui64> MsgCountByClusterSrc1 = {
- {"dc1", 14},
- {"dc2", 1}
- };
- THashMap<TString, ui64> MsgCountByClusterSrc2 = {
- {"dc1", 14},
- {"dc2", 6}
- };
- ui32 clustersPendingSrc1 = 2;
- ui32 clustersPendingSrc2 = 2;
-
- while (!stop && (clustersPendingSrc2 || clustersPendingSrc1)) {
- Cerr << "===Get event on client\n";
- auto event = *readSession->GetEvent(true);
- std::visit(TOverloaded {
- [&](TReadSessionEvent::TDataReceivedEvent& event) {
- Cerr << "===Data event\n";
- auto& clusterName = event.GetPartitionStream()->GetCluster();
- for (auto& message: event.GetMessages()) {
- TString sourceId = message.GetMessageGroupId();
- ui32 seqNo = message.GetSeqNo();
- if (sourceId == setup1->GetTestMessageGroupId()) {
- UNIT_ASSERT_VALUES_EQUAL(seqNo, seqNoByClusterSrc1[clusterName] + 1);
- seqNoByClusterSrc1[clusterName]++;
- auto& msgRemaining = MsgCountByClusterSrc1[clusterName];
- UNIT_ASSERT(msgRemaining > 0);
- msgRemaining--;
- if (!msgRemaining)
- clustersPendingSrc1--;
- } else {
- UNIT_ASSERT_VALUES_EQUAL(sourceId, tmpSrcId);
- auto& prevSeqNo = SeqNoByClusterSrc2[clusterName];
- if (clusterName == "dc1" || prevSeqNo != 1) {
- UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 1);
- prevSeqNo++;
- } else {
- UNIT_ASSERT_VALUES_EQUAL(clusterName, "dc2");
- UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 6);
- prevSeqNo+= 6;
- }
- auto& msgRemaining = MsgCountByClusterSrc2[clusterName];
- UNIT_ASSERT(msgRemaining > 0);
- msgRemaining--;
- if (!msgRemaining)
- clustersPendingSrc2--;
- }
- message.Commit();
- }
- },
- [&](TReadSessionEvent::TCommitAcknowledgementEvent&) {
- },
- [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) {
- event.Confirm();
- },
- [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) {
- event.Confirm();
- },
- [&](TReadSessionEvent::TPartitionStreamStatusEvent&) {
- Cerr << "===Status event\n";
- UNIT_FAIL("Test does not support lock sessions yet");
- },
- [&](TReadSessionEvent::TPartitionStreamClosedEvent&) {
- Cerr << "===Stream closed event\n";
- UNIT_FAIL("Test does not support lock sessions yet");
- },
- [&](TSessionClosedEvent& event) {
- Cerr << "===Got close event: " << event.DebugString();
- stop = true;
- }
-
- }, event);
- }
- UNIT_ASSERT_VALUES_EQUAL(clustersPendingSrc1 || clustersPendingSrc2, 0);
- }
- Y_UNIT_TEST(RetryWithBatching) {
- auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
- auto settings = setup->GetWriteSessionSettings();
- auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>();
- settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time.
- settings.BatchFlushSizeBytes(100);
- settings.RetryPolicy(retryPolicy);
- auto& client = setup->GetPersQueueClient();
- auto writer = client.CreateWriteSession(settings);
- auto event = *writer->GetEvent(true);
+ auto writer = client.CreateWriteSession(settings);
+ auto seqNo = writer->GetInitSeqNo().GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(seqNo, expectedSeqNo);
+ writer->Close(TDuration::Zero());
+ };
+
+ //!check SeqNo in both DC. For writer1 We expect 14 messages in DC1
+ //! (10 written initially + 4 wriiten after recoonect) and 1 message in DC2 (only initial message).
+ Cerr << "===Check SeqNo writer1, dc2\n";
+ CheckSeqNo("dc2", 1);
+ Cerr << "===Check SeqNo writer1, dc1\n";
+ CheckSeqNo("dc1", 14);
+
+ //! Check SeqNo for writer 2; Expect to have 6 messages on DC2 with MaxSeqNo = 6;
+ helper2 = nullptr;
+ settings.MessageGroupId(tmpSrcId);
+ Cerr << "===Check SeqNo writer2 dc2\n";
+ //!DC2 has no a ap in SeqNo since 5 messages were written ot dc 1.
+ CheckSeqNo("dc2", 11);
+ Cerr << "===Check SeqNo writer2 dc1\n";
+ CheckSeqNo("dc1", 14);
+
+
+ auto readSession = client.CreateReadSession(setup1->GetReadSessionSettings());
+
+ bool stop = false;
+ THashMap<TString, ui64> seqNoByClusterSrc1 = {
+ {"dc1", 0},
+ {"dc2", 0}
+ };
+ auto SeqNoByClusterSrc2 = seqNoByClusterSrc1;
+
+ THashMap<TString, ui64> MsgCountByClusterSrc1 = {
+ {"dc1", 14},
+ {"dc2", 1}
+ };
+ THashMap<TString, ui64> MsgCountByClusterSrc2 = {
+ {"dc1", 14},
+ {"dc2", 6}
+ };
+ ui32 clustersPendingSrc1 = 2;
+ ui32 clustersPendingSrc2 = 2;
+
+ while (!stop && (clustersPendingSrc2 || clustersPendingSrc1)) {
+ Cerr << "===Get event on client\n";
+ auto event = *readSession->GetEvent(true);
+ std::visit(TOverloaded {
+ [&](TReadSessionEvent::TDataReceivedEvent& event) {
+ Cerr << "===Data event\n";
+ auto& clusterName = event.GetPartitionStream()->GetCluster();
+ for (auto& message: event.GetMessages()) {
+ TString sourceId = message.GetMessageGroupId();
+ ui32 seqNo = message.GetSeqNo();
+ if (sourceId == setup1->GetTestMessageGroupId()) {
+ UNIT_ASSERT_VALUES_EQUAL(seqNo, seqNoByClusterSrc1[clusterName] + 1);
+ seqNoByClusterSrc1[clusterName]++;
+ auto& msgRemaining = MsgCountByClusterSrc1[clusterName];
+ UNIT_ASSERT(msgRemaining > 0);
+ msgRemaining--;
+ if (!msgRemaining)
+ clustersPendingSrc1--;
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(sourceId, tmpSrcId);
+ auto& prevSeqNo = SeqNoByClusterSrc2[clusterName];
+ if (clusterName == "dc1" || prevSeqNo != 1) {
+ UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 1);
+ prevSeqNo++;
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(clusterName, "dc2");
+ UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 6);
+ prevSeqNo+= 6;
+ }
+ auto& msgRemaining = MsgCountByClusterSrc2[clusterName];
+ UNIT_ASSERT(msgRemaining > 0);
+ msgRemaining--;
+ if (!msgRemaining)
+ clustersPendingSrc2--;
+ }
+ message.Commit();
+ }
+ },
+ [&](TReadSessionEvent::TCommitAcknowledgementEvent&) {
+ },
+ [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) {
+ event.Confirm();
+ },
+ [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) {
+ event.Confirm();
+ },
+ [&](TReadSessionEvent::TPartitionStreamStatusEvent&) {
+ Cerr << "===Status event\n";
+ UNIT_FAIL("Test does not support lock sessions yet");
+ },
+ [&](TReadSessionEvent::TPartitionStreamClosedEvent&) {
+ Cerr << "===Stream closed event\n";
+ UNIT_FAIL("Test does not support lock sessions yet");
+ },
+ [&](TSessionClosedEvent& event) {
+ Cerr << "===Got close event: " << event.DebugString();
+ stop = true;
+ }
+
+ }, event);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(clustersPendingSrc1 || clustersPendingSrc2, 0);
+ }
+ Y_UNIT_TEST(RetryWithBatching) {
+ auto setup = std::make_shared<TPersQueueYdbSdkTestSetup>(TEST_CASE_NAME);
+ auto settings = setup->GetWriteSessionSettings();
+ auto retryPolicy = std::make_shared<TYdbPqTestRetryPolicy>();
+ settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time.
+ settings.BatchFlushSizeBytes(100);
+ settings.RetryPolicy(retryPolicy);
+ auto& client = setup->GetPersQueueClient();
+ auto writer = client.CreateWriteSession(settings);
+ auto event = *writer->GetEvent(true);
Cerr << NYdb::NPersQueue::DebugString(event) << "\n";
UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
- auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
- TString message = "1234567890";
- ui64 seqNo = 0;
+ auto continueToken = std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken);
+ TString message = "1234567890";
+ ui64 seqNo = 0;
setup->KickTablets();
- writer->Write(std::move(continueToken), message, ++seqNo);
- retryPolicy->ExpectBreakDown();
- retryPolicy->WaitForRetriesSync(3);
+ writer->Write(std::move(continueToken), message, ++seqNo);
+ retryPolicy->ExpectBreakDown();
+ retryPolicy->WaitForRetriesSync(3);
while (seqNo < 10) {
auto event = *writer->GetEvent(true);
Cerr << NYdb::NPersQueue::DebugString(event) << "\n";
UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event));
- writer->Write(
+ writer->Write(
std::move(std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event).ContinuationToken),
- message, ++seqNo
- );
- }
-
- setup->AllowTablets();
- retryPolicy->WaitForRepairSync();
- WaitMessagesAcked(writer, 1, seqNo);
- }
-};
-}; //NYdb::NPersQueue::NTests
+ message, ++seqNo
+ );
+ }
+
+ setup->AllowTablets();
+ retryPolicy->WaitForRepairSync();
+ WaitMessagesAcked(writer, 1, seqNo);
+ }
+};
+}; //NYdb::NPersQueue::NTests
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp
index d9208d2b6c..6a853b6fc5 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.cpp
@@ -1,33 +1,33 @@
-#include "ut_utils.h"
-
-namespace NYdb::NPersQueue::NTests {
-
-void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo) {
- THashSet<ui64> ackedSeqNo;
- while (ackedSeqNo.size() < endSeqNo - startSeqNo + 1) {
- auto event = *writer->GetEvent(true);
- if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) {
- continue;
- } else {
- UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event));
- for (auto& ack : std::get<TWriteSessionEvent::TAcksEvent>(event).Acks) {
- UNIT_ASSERT(!ackedSeqNo.contains(ack.SeqNo));
- UNIT_ASSERT(ack.SeqNo >= startSeqNo && ack.SeqNo <= endSeqNo);
- ackedSeqNo.insert(ack.SeqNo);
- }
- }
- }
-}
-
-TSimpleWriteSessionTestAdapter::TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session)
- : Session(session)
-{}
-
-ui64 TSimpleWriteSessionTestAdapter::GetAcquiredMessagesCount() const {
- if (Session->Writer)
- return Session->Writer->MessagesAcquired;
- else
- return 0;
-}
-
-}
+#include "ut_utils.h"
+
+namespace NYdb::NPersQueue::NTests {
+
+void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo) {
+ THashSet<ui64> ackedSeqNo;
+ while (ackedSeqNo.size() < endSeqNo - startSeqNo + 1) {
+ auto event = *writer->GetEvent(true);
+ if (std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) {
+ continue;
+ } else {
+ UNIT_ASSERT(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event));
+ for (auto& ack : std::get<TWriteSessionEvent::TAcksEvent>(event).Acks) {
+ UNIT_ASSERT(!ackedSeqNo.contains(ack.SeqNo));
+ UNIT_ASSERT(ack.SeqNo >= startSeqNo && ack.SeqNo <= endSeqNo);
+ ackedSeqNo.insert(ack.SeqNo);
+ }
+ }
+ }
+}
+
+TSimpleWriteSessionTestAdapter::TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session)
+ : Session(session)
+{}
+
+ui64 TSimpleWriteSessionTestAdapter::GetAcquiredMessagesCount() const {
+ if (Session->Writer)
+ return Session->Writer->MessagesAcquired;
+ else
+ return 0;
+}
+
+}
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h
index 8d70b6af4d..7fb219757b 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils.h
@@ -1,25 +1,25 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h>
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/write_session.h>
-
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-
-namespace NYdb::NPersQueue::NTests {
-
-class TPersQueueYdbSdkTestSetup : public ::NPersQueue::SDKTestSetup {
- THolder<NYdb::TDriver> Driver;
+
+using namespace NKikimr;
+using namespace NKikimr::NPersQueueTests;
+
+namespace NYdb::NPersQueue::NTests {
+
+class TPersQueueYdbSdkTestSetup : public ::NPersQueue::SDKTestSetup {
+ THolder<NYdb::TDriver> Driver;
THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient;
-
- TAdaptiveLock Lock;
-public:
- TPersQueueYdbSdkTestSetup(const TString& testCaseName, bool start = true)
- : SDKTestSetup(testCaseName, start)
- {}
-
+
+ TAdaptiveLock Lock;
+public:
+ TPersQueueYdbSdkTestSetup(const TString& testCaseName, bool start = true)
+ : SDKTestSetup(testCaseName, start)
+ {}
+
~TPersQueueYdbSdkTestSetup() {
if (PersQueueClient) {
PersQueueClient = nullptr;
@@ -31,387 +31,387 @@ public:
}
}
- NYdb::TDriver& GetDriver() {
- if (!Driver) {
- NYdb::TDriverConfig cfg;
- cfg.SetEndpoint(TStringBuilder() << "localhost:" << Server.GrpcPort);
- cfg.SetDatabase("/Root");
+ NYdb::TDriver& GetDriver() {
+ if (!Driver) {
+ NYdb::TDriverConfig cfg;
+ cfg.SetEndpoint(TStringBuilder() << "localhost:" << Server.GrpcPort);
+ cfg.SetDatabase("/Root");
cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG));
- Driver = MakeHolder<NYdb::TDriver>(cfg);
- }
- return *Driver;
- }
-
+ Driver = MakeHolder<NYdb::TDriver>(cfg);
+ }
+ return *Driver;
+ }
+
NYdb::NPersQueue::TPersQueueClient& GetPersQueueClient() {
- with_lock(Lock) {
- if (!PersQueueClient) {
+ with_lock(Lock) {
+ if (!PersQueueClient) {
PersQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(GetDriver());
- }
- return *PersQueueClient;
- }
- }
-
- NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() {
- NYdb::NPersQueue::TReadSessionSettings settings;
- settings
- .ConsumerName(GetTestClient())
- .AppendTopics(GetTestTopic());
- return settings;
- }
-
- NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() {
- TWriteSessionSettings settings;
+ }
+ return *PersQueueClient;
+ }
+ }
+
+ NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() {
+ NYdb::NPersQueue::TReadSessionSettings settings;
+ settings
+ .ConsumerName(GetTestClient())
+ .AppendTopics(GetTestTopic());
+ return settings;
+ }
+
+ NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() {
+ TWriteSessionSettings settings;
settings
.Path(GetTestTopic())
.MessageGroupId(GetTestMessageGroupId());
- return settings;
- }
-};
-
-struct TYDBClientEventLoop : public ::NPersQueue::IClientEventLoop {
-public:
- std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup;
- using TAcksCallback = std::function<void (const TVector<ui64>&)>;
-
- TYDBClientEventLoop(
- std::shared_ptr<TPersQueueYdbSdkTestSetup> setup,
- IRetryPolicy::TPtr retryPolicy = nullptr,
+ return settings;
+ }
+};
+
+struct TYDBClientEventLoop : public ::NPersQueue::IClientEventLoop {
+public:
+ std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup;
+ using TAcksCallback = std::function<void (const TVector<ui64>&)>;
+
+ TYDBClientEventLoop(
+ std::shared_ptr<TPersQueueYdbSdkTestSetup> setup,
+ IRetryPolicy::TPtr retryPolicy = nullptr,
IExecutor::TPtr compressExecutor = nullptr,
const TString& preferredCluster = TString(),
- const TString& sourceId = TString()
- )
- : IClientEventLoop()
- , Setup(setup)
- {
- Log = Setup->GetLog();
+ const TString& sourceId = TString()
+ )
+ : IClientEventLoop()
+ , Setup(setup)
+ {
+ Log = Setup->GetLog();
Thread = std::make_unique<TThread>([setup, retryPolicy, compressExecutor, preferredCluster, sourceId, this]() {
- auto writerConfig = Setup->GetWriteSessionSettings();
- writerConfig.MaxMemoryUsage(100_MB);
- if (!sourceId.empty()) {
- writerConfig.MessageGroupId(sourceId);
- }
- if (retryPolicy != nullptr)
- writerConfig.RetryPolicy(retryPolicy);
- if (compressExecutor != nullptr)
- writerConfig.CompressionExecutor(compressExecutor);
+ auto writerConfig = Setup->GetWriteSessionSettings();
+ writerConfig.MaxMemoryUsage(100_MB);
+ if (!sourceId.empty()) {
+ writerConfig.MessageGroupId(sourceId);
+ }
+ if (retryPolicy != nullptr)
+ writerConfig.RetryPolicy(retryPolicy);
+ if (compressExecutor != nullptr)
+ writerConfig.CompressionExecutor(compressExecutor);
if (preferredCluster)
writerConfig.PreferredCluster(preferredCluster);
- auto writer = setup->GetPersQueueClient().CreateWriteSession(writerConfig);
-
- TMaybe<TContinuationToken> continueToken;
- NThreading::TFuture<void> waitEventFuture = writer->WaitEvent();
- THashMap<ui64, NThreading::TPromise<::NPersQueue::TWriteResult>> ackPromiseBySequenceNumber;
- while (!MustStop) {
- if (!continueToken) {
- Log << TLOG_INFO << "Wait for writer event";
- waitEventFuture.Wait();
- }
-
- bool closed = false;
- while (waitEventFuture.HasValue() && !closed) {
- TWriteSessionEvent::TEvent event = *writer->GetEvent(true);
- waitEventFuture = writer->WaitEvent();
- std::visit(TOverloaded {
- [&](const TWriteSessionEvent::TAcksEvent& event) {
- TVector<ui64> sequenceNumbers;
- for (const auto& ack : event.Acks) {
- UNIT_ASSERT(ackPromiseBySequenceNumber.contains(ack.SeqNo));
- sequenceNumbers.push_back(ack.SeqNo);
- ackPromiseBySequenceNumber[ack.SeqNo].SetValue({true, false});
- ackPromiseBySequenceNumber.erase(ack.SeqNo);
- }
- },
- [&](TWriteSessionEvent::TReadyToAcceptEvent& event) {
- Log << TLOG_INFO << "Got new continue token";
- continueToken = std::move(event.ContinuationToken);
- },
- [&](const TSessionClosedEvent& event) {
- Log << TLOG_INFO << "===Got close event: " << event.DebugString() << Endl;
- if (!MayStop) {
- UNIT_ASSERT(MustStop);
- UNIT_ASSERT(MessageBuffer.IsEmpty());
- UNIT_ASSERT(ackPromiseBySequenceNumber.empty());
- } else {
- MustStop = true;
- closed = true;
- }
- }
- }, event);
- }
-
- if (continueToken && !MessageBuffer.IsEmpty()) {
- ::NPersQueue::TAcknowledgableMessage acknowledgeableMessage;
- Y_VERIFY(MessageBuffer.Dequeue(acknowledgeableMessage));
- ackPromiseBySequenceNumber.emplace(acknowledgeableMessage.SequenceNumber, acknowledgeableMessage.AckPromise);
- Y_VERIFY(continueToken);
- Log << TLOG_INFO << "Write messages with sequence numbers " << acknowledgeableMessage.SequenceNumber;
- writer->Write(
- std::move(*continueToken),
- std::move(acknowledgeableMessage.Value),
- acknowledgeableMessage.SequenceNumber,
- acknowledgeableMessage.CreatedAt
- );
- continueToken = Nothing();
- }
- }
- Log << TLOG_DEBUG << "Close writer (stop)";
- writer->Close(TDuration::Zero());
- writer = nullptr;
- Log << TLOG_DEBUG << "Writer closed";
- });
- Thread->Start();
- }
-};
-
-struct TYdbPqTestRetryState : IRetryState {
- TYdbPqTestRetryState(
- std::function<void ()> retryCallback, std::function<void ()> destroyCallback, const TDuration& delay
- )
- : RetryDone(retryCallback)
- , DestroyDone(destroyCallback)
- , Delay(delay)
- {}
-
+ auto writer = setup->GetPersQueueClient().CreateWriteSession(writerConfig);
+
+ TMaybe<TContinuationToken> continueToken;
+ NThreading::TFuture<void> waitEventFuture = writer->WaitEvent();
+ THashMap<ui64, NThreading::TPromise<::NPersQueue::TWriteResult>> ackPromiseBySequenceNumber;
+ while (!MustStop) {
+ if (!continueToken) {
+ Log << TLOG_INFO << "Wait for writer event";
+ waitEventFuture.Wait();
+ }
+
+ bool closed = false;
+ while (waitEventFuture.HasValue() && !closed) {
+ TWriteSessionEvent::TEvent event = *writer->GetEvent(true);
+ waitEventFuture = writer->WaitEvent();
+ std::visit(TOverloaded {
+ [&](const TWriteSessionEvent::TAcksEvent& event) {
+ TVector<ui64> sequenceNumbers;
+ for (const auto& ack : event.Acks) {
+ UNIT_ASSERT(ackPromiseBySequenceNumber.contains(ack.SeqNo));
+ sequenceNumbers.push_back(ack.SeqNo);
+ ackPromiseBySequenceNumber[ack.SeqNo].SetValue({true, false});
+ ackPromiseBySequenceNumber.erase(ack.SeqNo);
+ }
+ },
+ [&](TWriteSessionEvent::TReadyToAcceptEvent& event) {
+ Log << TLOG_INFO << "Got new continue token";
+ continueToken = std::move(event.ContinuationToken);
+ },
+ [&](const TSessionClosedEvent& event) {
+ Log << TLOG_INFO << "===Got close event: " << event.DebugString() << Endl;
+ if (!MayStop) {
+ UNIT_ASSERT(MustStop);
+ UNIT_ASSERT(MessageBuffer.IsEmpty());
+ UNIT_ASSERT(ackPromiseBySequenceNumber.empty());
+ } else {
+ MustStop = true;
+ closed = true;
+ }
+ }
+ }, event);
+ }
+
+ if (continueToken && !MessageBuffer.IsEmpty()) {
+ ::NPersQueue::TAcknowledgableMessage acknowledgeableMessage;
+ Y_VERIFY(MessageBuffer.Dequeue(acknowledgeableMessage));
+ ackPromiseBySequenceNumber.emplace(acknowledgeableMessage.SequenceNumber, acknowledgeableMessage.AckPromise);
+ Y_VERIFY(continueToken);
+ Log << TLOG_INFO << "Write messages with sequence numbers " << acknowledgeableMessage.SequenceNumber;
+ writer->Write(
+ std::move(*continueToken),
+ std::move(acknowledgeableMessage.Value),
+ acknowledgeableMessage.SequenceNumber,
+ acknowledgeableMessage.CreatedAt
+ );
+ continueToken = Nothing();
+ }
+ }
+ Log << TLOG_DEBUG << "Close writer (stop)";
+ writer->Close(TDuration::Zero());
+ writer = nullptr;
+ Log << TLOG_DEBUG << "Writer closed";
+ });
+ Thread->Start();
+ }
+};
+
+struct TYdbPqTestRetryState : IRetryState {
+ TYdbPqTestRetryState(
+ std::function<void ()> retryCallback, std::function<void ()> destroyCallback, const TDuration& delay
+ )
+ : RetryDone(retryCallback)
+ , DestroyDone(destroyCallback)
+ , Delay(delay)
+ {}
+
TMaybe<TDuration> GetNextRetryDelay(const TStatus&) override {
- Cerr << "Test retry state: get retry delay\n";
- RetryDone();
- return Delay;
- }
- std::function<void ()> RetryDone;
- std::function<void ()> DestroyDone;
- TDuration Delay;
-
- ~TYdbPqTestRetryState() {
- DestroyDone();
- }
-};
-struct TYdbPqNoRetryState : IRetryState {
- TAtomic DelayCalled = 0;
+ Cerr << "Test retry state: get retry delay\n";
+ RetryDone();
+ return Delay;
+ }
+ std::function<void ()> RetryDone;
+ std::function<void ()> DestroyDone;
+ TDuration Delay;
+
+ ~TYdbPqTestRetryState() {
+ DestroyDone();
+ }
+};
+struct TYdbPqNoRetryState : IRetryState {
+ TAtomic DelayCalled = 0;
TMaybe<TDuration> GetNextRetryDelay(const TStatus&) override {
- auto res = AtomicSwap(&DelayCalled, 0);
- UNIT_ASSERT(!res);
- return Nothing();
- }
-};
-
-struct TYdbPqTestRetryPolicy : IRetryPolicy {
- TYdbPqTestRetryPolicy(const TDuration& delay = TDuration::MilliSeconds(2000))
- : Delay(delay)
- {}
-
- IRetryState::TPtr CreateRetryState() const override {
- if (AtomicSwap(&OnFatalBreakDown, 0)) {
- return std::make_unique<TYdbPqNoRetryState>();
- }
- if (AtomicGet(Initialized_)) {
- auto res = AtomicSwap(&OnBreakDown, 0);
- UNIT_ASSERT(res);
- UNIT_ASSERT(AtomicGet(CurrentRetries) == 0);
- }
- auto retryCb = [this]() mutable {this->RetryDone();};
- auto destroyCb = [this]() mutable {this->StateDestroyed();};
- return std::make_unique<TYdbPqTestRetryState>(retryCb, destroyCb, Delay);
- }
-
- void RetryDone() const {
- AtomicAdd(CurrentRetries, 1);
- auto expected = AtomicGet(RetriesExpected);
- if (expected > 0 && AtomicGet(CurrentRetries) >= expected) {
- with_lock(Lock) {
- RetryPromise.SetValue();
- }
- AtomicSet(RetriesExpected, 0);
- }
- }
- void StateDestroyed() const {
- auto expected = AtomicSwap(&RepairExpected, 0);
- if (expected) {
- with_lock(Lock) {
- RepairPromise.SetValue();
- }
- }
- }
- void ExpectBreakDown() {
- UNIT_ASSERT(AtomicGet(OnBreakDown) == 0);
- AtomicSet(CurrentRetries, 0);
- AtomicSet(OnBreakDown, 1);
- }
- void ExpectFatalBreakDown() {
- AtomicSet(OnFatalBreakDown, 1);
- }
-
- void WaitForRetries(ui64 retryCount, NThreading::TPromise<void>& promise) {
- AtomicSet(RetriesExpected, retryCount);
- with_lock(Lock) {
- RetryPromise = promise;
- }
- }
- void WaitForRetriesSync(ui64 retryCount) {
- NThreading::TPromise<void> retriesPromise = NThreading::NewPromise();
- auto retriesFuture = retriesPromise.GetFuture();
- WaitForRetries(retryCount, retriesPromise);
- retriesFuture.Wait();
- }
-
- void WaitForRepair(NThreading::TPromise<void>& promise) {
- AtomicSet(RepairExpected, 1 );
- with_lock(Lock) {
- RepairPromise = promise;
- }
- }
-
- void WaitForRepairSync() {
- NThreading::TPromise<void> repairPromise = NThreading::NewPromise();
- auto repairFuture = repairPromise.GetFuture();
- WaitForRepair(repairPromise);
- repairFuture.Wait();
- }
-
- void Initialized() {
- AtomicSet(Initialized_, 1);
- AtomicSet(CurrentRetries, 0);
- }
-private:
- TDuration Delay;
- mutable TAtomic CurrentRetries = 0;
- mutable TAtomic Initialized_ = 0;
- mutable TAtomic OnBreakDown = 0;
- mutable TAtomic OnFatalBreakDown = 0;
- mutable NThreading::TPromise<void> RetryPromise;
- mutable NThreading::TPromise<void> RepairPromise;
- mutable TAtomic RetriesExpected = 0;
- mutable TAtomic RepairExpected = 0;
- mutable TAdaptiveLock Lock;
-};
-
-class TYdbPqTestExecutor : public IAsyncExecutor {
-public:
- TYdbPqTestExecutor(std::shared_ptr<TLockFreeQueue<ui64>> idsQueue)
- : Stop()
- , ExecIdsQueue(idsQueue)
- , Thread([idsQueue, this]() {
- while(!Stop) {
- TFunction f;
- while (TasksQueue.Dequeue(&f)) {
- ++CurrentTaskId;
- Cerr << "Enqueue task with id " << CurrentTaskId << Endl;
- Tasks[CurrentTaskId] = f;
- }
- ui64 id = 0;
- while (ExecIdsQueue->Dequeue(&id)) {
- ExecIds.push(id);
- Cerr << "Got ok to execute task with id " << id << Endl;
-
- }
- while (!ExecIds.empty()) {
- auto id = ExecIds.front();
- auto iter = Tasks.find(id);
- if (iter == Tasks.end())
- break;
- Cerr << "Executing compression of " << id << Endl;
- ExecIds.pop();
- try {
- (iter->second)();
- } catch (...) {
- Cerr << "Failed on compression call: " << CurrentExceptionMessage() << Endl;
- Y_FAIL();
- }
- Cerr << "Compression of " << id << " Done\n";
- Tasks.erase(iter);
- }
-
- }
- })
- {
- }
- ~TYdbPqTestExecutor() {
- Stop = true;
- Thread.Join();
- }
- void PostImpl(TVector<TFunction>&& fs) override {
- for (auto& f : fs) {
- TasksQueue.Enqueue(std::move(f));
- }
- }
-
- void PostImpl(TFunction&& f) override {
- TasksQueue.Enqueue(std::move(f));
- }
-
+ auto res = AtomicSwap(&DelayCalled, 0);
+ UNIT_ASSERT(!res);
+ return Nothing();
+ }
+};
+
+struct TYdbPqTestRetryPolicy : IRetryPolicy {
+ TYdbPqTestRetryPolicy(const TDuration& delay = TDuration::MilliSeconds(2000))
+ : Delay(delay)
+ {}
+
+ IRetryState::TPtr CreateRetryState() const override {
+ if (AtomicSwap(&OnFatalBreakDown, 0)) {
+ return std::make_unique<TYdbPqNoRetryState>();
+ }
+ if (AtomicGet(Initialized_)) {
+ auto res = AtomicSwap(&OnBreakDown, 0);
+ UNIT_ASSERT(res);
+ UNIT_ASSERT(AtomicGet(CurrentRetries) == 0);
+ }
+ auto retryCb = [this]() mutable {this->RetryDone();};
+ auto destroyCb = [this]() mutable {this->StateDestroyed();};
+ return std::make_unique<TYdbPqTestRetryState>(retryCb, destroyCb, Delay);
+ }
+
+ void RetryDone() const {
+ AtomicAdd(CurrentRetries, 1);
+ auto expected = AtomicGet(RetriesExpected);
+ if (expected > 0 && AtomicGet(CurrentRetries) >= expected) {
+ with_lock(Lock) {
+ RetryPromise.SetValue();
+ }
+ AtomicSet(RetriesExpected, 0);
+ }
+ }
+ void StateDestroyed() const {
+ auto expected = AtomicSwap(&RepairExpected, 0);
+ if (expected) {
+ with_lock(Lock) {
+ RepairPromise.SetValue();
+ }
+ }
+ }
+ void ExpectBreakDown() {
+ UNIT_ASSERT(AtomicGet(OnBreakDown) == 0);
+ AtomicSet(CurrentRetries, 0);
+ AtomicSet(OnBreakDown, 1);
+ }
+ void ExpectFatalBreakDown() {
+ AtomicSet(OnFatalBreakDown, 1);
+ }
+
+ void WaitForRetries(ui64 retryCount, NThreading::TPromise<void>& promise) {
+ AtomicSet(RetriesExpected, retryCount);
+ with_lock(Lock) {
+ RetryPromise = promise;
+ }
+ }
+ void WaitForRetriesSync(ui64 retryCount) {
+ NThreading::TPromise<void> retriesPromise = NThreading::NewPromise();
+ auto retriesFuture = retriesPromise.GetFuture();
+ WaitForRetries(retryCount, retriesPromise);
+ retriesFuture.Wait();
+ }
+
+ void WaitForRepair(NThreading::TPromise<void>& promise) {
+ AtomicSet(RepairExpected, 1 );
+ with_lock(Lock) {
+ RepairPromise = promise;
+ }
+ }
+
+ void WaitForRepairSync() {
+ NThreading::TPromise<void> repairPromise = NThreading::NewPromise();
+ auto repairFuture = repairPromise.GetFuture();
+ WaitForRepair(repairPromise);
+ repairFuture.Wait();
+ }
+
+ void Initialized() {
+ AtomicSet(Initialized_, 1);
+ AtomicSet(CurrentRetries, 0);
+ }
+private:
+ TDuration Delay;
+ mutable TAtomic CurrentRetries = 0;
+ mutable TAtomic Initialized_ = 0;
+ mutable TAtomic OnBreakDown = 0;
+ mutable TAtomic OnFatalBreakDown = 0;
+ mutable NThreading::TPromise<void> RetryPromise;
+ mutable NThreading::TPromise<void> RepairPromise;
+ mutable TAtomic RetriesExpected = 0;
+ mutable TAtomic RepairExpected = 0;
+ mutable TAdaptiveLock Lock;
+};
+
+class TYdbPqTestExecutor : public IAsyncExecutor {
+public:
+ TYdbPqTestExecutor(std::shared_ptr<TLockFreeQueue<ui64>> idsQueue)
+ : Stop()
+ , ExecIdsQueue(idsQueue)
+ , Thread([idsQueue, this]() {
+ while(!Stop) {
+ TFunction f;
+ while (TasksQueue.Dequeue(&f)) {
+ ++CurrentTaskId;
+ Cerr << "Enqueue task with id " << CurrentTaskId << Endl;
+ Tasks[CurrentTaskId] = f;
+ }
+ ui64 id = 0;
+ while (ExecIdsQueue->Dequeue(&id)) {
+ ExecIds.push(id);
+ Cerr << "Got ok to execute task with id " << id << Endl;
+
+ }
+ while (!ExecIds.empty()) {
+ auto id = ExecIds.front();
+ auto iter = Tasks.find(id);
+ if (iter == Tasks.end())
+ break;
+ Cerr << "Executing compression of " << id << Endl;
+ ExecIds.pop();
+ try {
+ (iter->second)();
+ } catch (...) {
+ Cerr << "Failed on compression call: " << CurrentExceptionMessage() << Endl;
+ Y_FAIL();
+ }
+ Cerr << "Compression of " << id << " Done\n";
+ Tasks.erase(iter);
+ }
+
+ }
+ })
+ {
+ }
+ ~TYdbPqTestExecutor() {
+ Stop = true;
+ Thread.Join();
+ }
+ void PostImpl(TVector<TFunction>&& fs) override {
+ for (auto& f : fs) {
+ TasksQueue.Enqueue(std::move(f));
+ }
+ }
+
+ void PostImpl(TFunction&& f) override {
+ TasksQueue.Enqueue(std::move(f));
+ }
+
void DoStart() override {
Thread.Start();
}
-private:
- std::atomic_bool Stop;
- TLockFreeQueue<TFunction> TasksQueue;
- std::shared_ptr<TLockFreeQueue<ui64>> ExecIdsQueue;
- THashMap<ui64, TFunction> Tasks;
- TQueue<ui64> ExecIds;
- ui64 CurrentTaskId = 0;
- TThread Thread;
-
-};
-
-
-struct TYdbPqWriterTestHelper {
- std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup;
- std::shared_ptr<TYdbPqTestRetryPolicy> Policy;
- std::unique_ptr<TYDBClientEventLoop> EventLoop;
- TIntrusivePtr<TYdbPqTestExecutor> CompressExecutor;
-
- TAutoEvent MessagesWrittenToBuffer;
- ui64 SeqNo = 1;
- TString Message = "message";
-public:
- TYdbPqWriterTestHelper(
- const TString& name,
- std::shared_ptr<TLockFreeQueue<ui64>> executorQueue = nullptr,
+private:
+ std::atomic_bool Stop;
+ TLockFreeQueue<TFunction> TasksQueue;
+ std::shared_ptr<TLockFreeQueue<ui64>> ExecIdsQueue;
+ THashMap<ui64, TFunction> Tasks;
+ TQueue<ui64> ExecIds;
+ ui64 CurrentTaskId = 0;
+ TThread Thread;
+
+};
+
+
+struct TYdbPqWriterTestHelper {
+ std::shared_ptr<TPersQueueYdbSdkTestSetup> Setup;
+ std::shared_ptr<TYdbPqTestRetryPolicy> Policy;
+ std::unique_ptr<TYDBClientEventLoop> EventLoop;
+ TIntrusivePtr<TYdbPqTestExecutor> CompressExecutor;
+
+ TAutoEvent MessagesWrittenToBuffer;
+ ui64 SeqNo = 1;
+ TString Message = "message";
+public:
+ TYdbPqWriterTestHelper(
+ const TString& name,
+ std::shared_ptr<TLockFreeQueue<ui64>> executorQueue = nullptr,
const TString& preferredCluster = TString(),
- std::shared_ptr<TPersQueueYdbSdkTestSetup> setup = nullptr,
- const TString& sourceId = TString()
- )
- : Setup(setup ? setup : std::make_shared<TPersQueueYdbSdkTestSetup>(name))
- , Policy(std::make_shared<TYdbPqTestRetryPolicy>())
- {
- if (executorQueue)
- CompressExecutor = MakeIntrusive<TYdbPqTestExecutor>(executorQueue);
+ std::shared_ptr<TPersQueueYdbSdkTestSetup> setup = nullptr,
+ const TString& sourceId = TString()
+ )
+ : Setup(setup ? setup : std::make_shared<TPersQueueYdbSdkTestSetup>(name))
+ , Policy(std::make_shared<TYdbPqTestRetryPolicy>())
+ {
+ if (executorQueue)
+ CompressExecutor = MakeIntrusive<TYdbPqTestExecutor>(executorQueue);
EventLoop = std::make_unique<TYDBClientEventLoop>(Setup, Policy, CompressExecutor, preferredCluster, sourceId);
- }
-
- NThreading::TFuture<::NPersQueue::TWriteResult> Write(bool doWait = false, const TString& message = TString()) {
- //auto f = ClientWrite(Message, SeqNo, TInstant::Now());
- auto promise = NThreading::NewPromise<::NPersQueue::TWriteResult>();
- auto log = Setup->GetLog();
- log << TLOG_INFO << "Enqueue message with sequence number " << SeqNo;
- EventLoop->MessageBuffer.Enqueue(::NPersQueue::TAcknowledgableMessage{
- message.Empty() ? Message : message,
- SeqNo, TInstant::Now(), promise
- });
- MessagesWrittenToBuffer.Signal();
- auto f = promise.GetFuture();
- ++SeqNo;
- if (doWait)
- f.Wait();
- return f;
- }
- ~TYdbPqWriterTestHelper() {
- EventLoop = nullptr;
- Setup = nullptr;
- CompressExecutor = nullptr;
- Policy = nullptr;
- }
-};
-
-class TSimpleWriteSessionTestAdapter {
-public:
- TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session);
- ui64 GetAcquiredMessagesCount() const;
-
-private:
- TSimpleBlockingWriteSession* Session;
-};
-
-
-void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo);
-} // namespace NYdb::NPersQueue::NTests
+ }
+
+ NThreading::TFuture<::NPersQueue::TWriteResult> Write(bool doWait = false, const TString& message = TString()) {
+ //auto f = ClientWrite(Message, SeqNo, TInstant::Now());
+ auto promise = NThreading::NewPromise<::NPersQueue::TWriteResult>();
+ auto log = Setup->GetLog();
+ log << TLOG_INFO << "Enqueue message with sequence number " << SeqNo;
+ EventLoop->MessageBuffer.Enqueue(::NPersQueue::TAcknowledgableMessage{
+ message.Empty() ? Message : message,
+ SeqNo, TInstant::Now(), promise
+ });
+ MessagesWrittenToBuffer.Signal();
+ auto f = promise.GetFuture();
+ ++SeqNo;
+ if (doWait)
+ f.Wait();
+ return f;
+ }
+ ~TYdbPqWriterTestHelper() {
+ EventLoop = nullptr;
+ Setup = nullptr;
+ CompressExecutor = nullptr;
+ Policy = nullptr;
+ }
+};
+
+class TSimpleWriteSessionTestAdapter {
+public:
+ TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session);
+ ui64 GetAcquiredMessagesCount() const;
+
+private:
+ TSimpleBlockingWriteSession* Session;
+};
+
+
+void WaitMessagesAcked(std::shared_ptr<IWriteSession> writer, ui64 startSeqNo, ui64 endSeqNo);
+} // namespace NYdb::NPersQueue::NTests
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h
index 81d915bd83..677eb8c03d 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/sdk_test_setup.h
@@ -2,7 +2,7 @@
#include "test_server.h"
#include <ydb/library/persqueue/topic_parser_public/topic_parser.h>
#include <library/cpp/logger/log.h>
-#include <util/system/tempfile.h>
+#include <util/system/tempfile.h>
#define TEST_CASE_NAME (this->Name_)
@@ -13,8 +13,8 @@ protected:
TString TestCaseName;
THolder<TTempFileHandle> NetDataFile;
- THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> DataCenters;
- TString LocalDC = "dc1";
+ THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> DataCenters;
+ TString LocalDC = "dc1";
TTestServer Server = TTestServer(false /* don't start */);
TLog Log = TLog("cerr");
@@ -39,10 +39,10 @@ public:
// 2. 'got more than 10 uncommitted reads'
Server.ServerSettings.PQConfig.Clear();
Server.ServerSettings.PQConfig.SetEnabled(true);
- Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1);
- Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1);
+ Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1);
+ Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1);
Server.ServerSettings.PQClusterDiscoveryConfig.SetEnabled(true);
- SetNetDataViaFile("::1/128\t" + GetLocalCluster());
+ SetNetDataViaFile("::1/128\t" + GetLocalCluster());
auto seed = TInstant::Now().MicroSeconds();
// This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable
@@ -51,7 +51,7 @@ public:
}
void Start(bool waitInit = true, bool addBrokenDatacenter = false) {
- Server.StartServer(false);
+ Server.StartServer(false);
//Server.EnableLogs({NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY});
Server.AnnoyingClient->InitRoot();
if (DataCenters.empty()) {
@@ -62,10 +62,10 @@ public:
}
Server.AnnoyingClient->InitDCs(dataCenters);
} else {
- Server.AnnoyingClient->InitDCs(DataCenters, LocalDC);
+ Server.AnnoyingClient->InitDCs(DataCenters, LocalDC);
}
Server.AnnoyingClient->InitSourceIds();
- CreateTopic(GetTestTopic(), GetLocalCluster());
+ CreateTopic(GetTestTopic(), GetLocalCluster());
if (waitInit) {
Server.WaitInit(GetTestTopic());
}
@@ -84,7 +84,7 @@ public:
}
TString GetLocalCluster() const {
- return LocalDC;
+ return LocalDC;
}
ui16 GetGrpcPort() const {
@@ -95,13 +95,13 @@ public:
return Server.GrpcServerOptions;
}
- void SetNetDataViaFile(const TString& netDataTsv) {
- NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv");
- NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size());
- NetDataFile->FlushData();
- Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name());
- }
-
+ void SetNetDataViaFile(const TString& netDataTsv) {
+ NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv");
+ NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size());
+ NetDataFile->FlushData();
+ Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name());
+ }
+
TLog& GetLog() {
return Log;
@@ -131,40 +131,40 @@ public:
}
void SetSingleDataCenter(const TString& name = "dc1") {
- UNIT_ASSERT(
- DataCenters.insert(std::make_pair(
- name,
- NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true}
- )).second
- );
- LocalDC = name;
+ UNIT_ASSERT(
+ DataCenters.insert(std::make_pair(
+ name,
+ NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true}
+ )).second
+ );
+ LocalDC = name;
}
void AddDataCenter(const TString& name, const TString& address, bool enabled = true, bool setSelfAsDc = true) {
if (DataCenters.empty() && setSelfAsDc) {
SetSingleDataCenter();
}
- NKikimr::NPersQueueTests::TPQTestClusterInfo info{
+ NKikimr::NPersQueueTests::TPQTestClusterInfo info{
address,
- enabled
- };
- UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second);
+ enabled
+ };
+ UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second);
}
void AddDataCenter(const TString& name, const SDKTestSetup& cluster, bool enabled = true, bool setSelfAsDc = true) {
AddDataCenter(name, TStringBuilder() << "localhost:" << cluster.Server.GrpcPort, enabled, setSelfAsDc);
}
- void EnableDataCenter(const TString& name) {
- auto iter = DataCenters.find(name);
- UNIT_ASSERT(iter != DataCenters.end());
- Server.AnnoyingClient->UpdateDcEnabled(name, true);
- }
- void DisableDataCenter(const TString& name) {
- auto iter = DataCenters.find(name);
- UNIT_ASSERT(iter != DataCenters.end());
- Server.AnnoyingClient->UpdateDcEnabled(name, false);
- }
+ void EnableDataCenter(const TString& name) {
+ auto iter = DataCenters.find(name);
+ UNIT_ASSERT(iter != DataCenters.end());
+ Server.AnnoyingClient->UpdateDcEnabled(name, true);
+ }
+ void DisableDataCenter(const TString& name) {
+ auto iter = DataCenters.find(name);
+ UNIT_ASSERT(iter != DataCenters.end());
+ Server.AnnoyingClient->UpdateDcEnabled(name, false);
+ }
void ShutdownGRpc() {
Server.ShutdownGRpc();
@@ -174,21 +174,21 @@ public:
Server.EnableGRpc();
Server.WaitInit(GetTestTopic());
}
-
- void KickTablets() {
+
+ void KickTablets() {
for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) {
- Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false);
+ Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false);
}
for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) {
- Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i);
- }
- }
- void AllowTablets() {
+ Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i);
+ }
+ }
+ void AllowTablets() {
for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) {
- Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true);
- }
- }
-
+ Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true);
+ }
+ }
+
void CreateTopic(const TString& topic, const TString& cluster, size_t partitionsCount = 1) {
Server.AnnoyingClient->CreateTopic(BuildFullTopicName(topic, cluster), partitionsCount);
}
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h
index 0b4a58d483..250ce03620 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_server.h
@@ -11,43 +11,43 @@
namespace NPersQueue {
-static constexpr int DEBUG_LOG_LEVEL = 7;
-
+static constexpr int DEBUG_LOG_LEVEL = 7;
+
class TTestServer {
public:
- TTestServer(bool start = true, TMaybe<TSimpleSharedPtr<TPortManager>> portManager = Nothing())
- : PortManager(portManager.GetOrElse(MakeSimpleShared<TPortManager>()))
- , Port(PortManager->GetPort(2134))
- , GrpcPort(PortManager->GetPort(2135))
- , ServerSettings(NKikimr::NPersQueueTests::PQSettings(Port).SetGrpcPort(GrpcPort))
+ TTestServer(bool start = true, TMaybe<TSimpleSharedPtr<TPortManager>> portManager = Nothing())
+ : PortManager(portManager.GetOrElse(MakeSimpleShared<TPortManager>()))
+ , Port(PortManager->GetPort(2134))
+ , GrpcPort(PortManager->GetPort(2135))
+ , ServerSettings(NKikimr::NPersQueueTests::PQSettings(Port).SetGrpcPort(GrpcPort))
, GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort))
{
if (start) {
StartServer();
}
}
- TTestServer(const NKikimr::Tests::TServerSettings& settings, bool start = true)
- : PortManager(MakeSimpleShared<TPortManager>())
- , Port(PortManager->GetPort(2134))
- , GrpcPort(PortManager->GetPort(2135))
- , ServerSettings(settings)
- , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort))
- {
- ServerSettings.Port = Port;
- ServerSettings.SetGrpcPort(GrpcPort);
- if (start)
- StartServer();
- }
-
- void StartServer(bool doClientInit = true) {
- PrepareNetDataFile();
+ TTestServer(const NKikimr::Tests::TServerSettings& settings, bool start = true)
+ : PortManager(MakeSimpleShared<TPortManager>())
+ , Port(PortManager->GetPort(2134))
+ , GrpcPort(PortManager->GetPort(2135))
+ , ServerSettings(settings)
+ , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort))
+ {
+ ServerSettings.Port = Port;
+ ServerSettings.SetGrpcPort(GrpcPort);
+ if (start)
+ StartServer();
+ }
+
+ void StartServer(bool doClientInit = true) {
+ PrepareNetDataFile();
CleverServer = MakeHolder<NKikimr::Tests::TServer>(ServerSettings);
CleverServer->EnableGRpc(GrpcServerOptions);
AnnoyingClient = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(ServerSettings, GrpcPort);
EnableLogs(LOGGED_SERVICES);
- if (doClientInit) {
- AnnoyingClient->FullInit();
- }
+ if (doClientInit) {
+ AnnoyingClient->FullInit();
+ }
}
void ShutdownGRpc() {
@@ -68,7 +68,7 @@ public:
}
void EnableLogs(const TVector<NKikimrServices::EServiceKikimr> services,
- NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) {
+ NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) {
Y_VERIFY(CleverServer != nullptr, "Start server before enabling logs");
for (auto s : services) {
CleverServer->GetRuntime()->SetLogPriority(s, prio);
@@ -79,14 +79,14 @@ public:
AnnoyingClient->WaitTopicInit(topic);
}
- bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") {
- if (NetDataFile)
- return false;
+ bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") {
+ if (NetDataFile)
+ return false;
NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv");
NetDataFile->Write(content.Data(), content.Size());
NetDataFile->FlushData();
ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name());
- return true;
+ return true;
}
void UpdateDC(const TString& name, bool local, bool enabled) {
@@ -94,7 +94,7 @@ public:
}
public:
- TSimpleSharedPtr<TPortManager> PortManager;
+ TSimpleSharedPtr<TPortManager> PortManager;
ui16 Port;
ui16 GrpcPort;
@@ -105,7 +105,7 @@ public:
THolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient> AnnoyingClient;
-
+
static const TVector<NKikimrServices::EServiceKikimr> LOGGED_SERVICES;
};
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h
index cbb4d38886..f5841d6277 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/test_utils.h
@@ -1,74 +1,74 @@
#pragma once
#include <util/generic/ptr.h>
-#include <util/generic/size_literals.h>
-#include <library/cpp/threading/chunk_queue/queue.h>
+#include <util/generic/size_literals.h>
+#include <library/cpp/threading/chunk_queue/queue.h>
#include <util/generic/overloaded.h>
-#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/testing/unittest/registar.h>
#include "sdk_test_setup.h"
-
+
namespace NPersQueue {
-using namespace NThreading;
-using namespace NYdb::NPersQueue;
-using namespace NKikimr;
-using namespace NKikimr::NPersQueueTests;
-
+using namespace NThreading;
+using namespace NYdb::NPersQueue;
+using namespace NKikimr;
+using namespace NKikimr::NPersQueueTests;
+
+
+struct TWriteResult {
+ bool Ok = false;
+ // No acknowledgement is expected from a writer under test
+ bool NoWait = false;
+ TString ResponseDebugString = TString();
+};
+
+struct TAcknowledgableMessage {
+ TString Value;
+ ui64 SequenceNumber;
+ TInstant CreatedAt;
+ TPromise<TWriteResult> AckPromise;
+};
+
+class IClientEventLoop {
+protected:
+ std::atomic_bool MayStop;
+ std::atomic_bool MustStop;
+ bool Stopped = false;
+ std::unique_ptr<TThread> Thread;
+ TLog Log;
+
+public:
+ IClientEventLoop()
+ : MayStop()
+ , MustStop()
+ , MessageBuffer()
+ {}
+
+ void AllowStop() {
+ MayStop = true;
+ }
+
+ void WaitForStop() {
+ if (!Stopped) {
+ Log << TLOG_INFO << "Wait for writer to die on itself";
+ Thread->Join();
+ Log << TLOG_INFO << "Client write event loop stopped";
+ }
+ Stopped = true;
+ }
+
+ virtual ~IClientEventLoop() {
+ MustStop = true;
+ if (!Stopped) {
+ Log << TLOG_INFO << "Wait for client write event loop to stop";
+ Thread->Join();
+ Log << TLOG_INFO << "Client write event loop stopped";
+ }
+ Stopped = true;
+ }
+
+ TManyOneQueue<TAcknowledgableMessage> MessageBuffer;
+
+};
-struct TWriteResult {
- bool Ok = false;
- // No acknowledgement is expected from a writer under test
- bool NoWait = false;
- TString ResponseDebugString = TString();
-};
-
-struct TAcknowledgableMessage {
- TString Value;
- ui64 SequenceNumber;
- TInstant CreatedAt;
- TPromise<TWriteResult> AckPromise;
-};
-
-class IClientEventLoop {
-protected:
- std::atomic_bool MayStop;
- std::atomic_bool MustStop;
- bool Stopped = false;
- std::unique_ptr<TThread> Thread;
- TLog Log;
-
-public:
- IClientEventLoop()
- : MayStop()
- , MustStop()
- , MessageBuffer()
- {}
-
- void AllowStop() {
- MayStop = true;
- }
-
- void WaitForStop() {
- if (!Stopped) {
- Log << TLOG_INFO << "Wait for writer to die on itself";
- Thread->Join();
- Log << TLOG_INFO << "Client write event loop stopped";
- }
- Stopped = true;
- }
-
- virtual ~IClientEventLoop() {
- MustStop = true;
- if (!Stopped) {
- Log << TLOG_INFO << "Wait for client write event loop to stop";
- Thread->Join();
- Log << TLOG_INFO << "Client write event loop stopped";
- }
- Stopped = true;
- }
-
- TManyOneQueue<TAcknowledgableMessage> MessageBuffer;
-
-};
-
} // namespace NPersQueue
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make
index a08cab9ece..fc27259e6e 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut/ya.make
@@ -39,7 +39,7 @@ SRCS(
basic_usage_ut.cpp
compress_executor_ut.cpp
retry_policy_ut.cpp
- ut_utils.cpp
+ ut_utils.cpp
)
END()
diff --git a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make
index 178452d4e0..a95f5de06f 100644
--- a/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make
+++ b/ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ya.make
@@ -34,10 +34,10 @@ SRCS(
common_ut.cpp
read_session_ut.cpp
basic_usage_ut.cpp
- compress_executor_ut.cpp
+ compress_executor_ut.cpp
compression_ut.cpp
- retry_policy_ut.cpp
- ut_utils.cpp
+ retry_policy_ut.cpp
+ ut_utils.cpp
)
END()
diff --git a/ydb/public/sdk/python/ydb/connection.py b/ydb/public/sdk/python/ydb/connection.py
index 5bbffcc759..573b276e20 100644
--- a/ydb/public/sdk/python/ydb/connection.py
+++ b/ydb/public/sdk/python/ydb/connection.py
@@ -206,11 +206,11 @@ def _construct_channel_options(driver_config, endpoint_options=None):
if driver_config.channel_options is None:
return _default_connect_options
channel_options = copy.deepcopy(driver_config.channel_options)
- custom_options_keys = set(i[0] for i in driver_config.channel_options)
+ custom_options_keys = set(i[0] for i in driver_config.channel_options)
for item in filter(
lambda x: x[0] not in custom_options_keys, _default_connect_options
):
- channel_options.append(item)
+ channel_options.append(item)
return channel_options
@@ -297,8 +297,8 @@ def channel_factory(
):
channel_provider = channel_provider if channel_provider is not None else grpc
options = _construct_channel_options(driver_config, endpoint_options)
- logger.debug("Channel options: {}".format(options))
-
+ logger.debug("Channel options: {}".format(options))
+
if driver_config.root_certificates is None and not driver_config.secure_channel:
return channel_provider.insecure_channel(endpoint, options)
root_certificates = driver_config.root_certificates
diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp
index b0940265d2..2ede2a2d05 100644
--- a/ydb/services/datastreams/datastreams_ut.cpp
+++ b/ydb/services/datastreams/datastreams_ut.cpp
@@ -10,9 +10,9 @@
#include <ydb/public/sdk/cpp/client/ydb_scheme/scheme.h>
#include <ydb/public/api/grpc/draft/ydb_datastreams_v1.grpc.pb.h>
-#include <library/cpp/json/json_reader.h>
+#include <library/cpp/json/json_reader.h>
#include <library/cpp/digest/md5/md5.h>
-
+
#include <random>
@@ -30,22 +30,22 @@ using TKikimrWithGrpcAndRootSchemaSecure = NYdb::TBasicKikimrWithGrpcAndRootSche
template<class TKikimr, bool secure>
-class TDatastreamsTestServer {
+class TDatastreamsTestServer {
public:
- TDatastreamsTestServer() {
+ TDatastreamsTestServer() {
NKikimrConfig::TAppConfig appConfig;
appConfig.MutablePQConfig()->SetTopicsAreFirstClassCitizen(true);
appConfig.MutablePQConfig()->SetEnabled(true);
- appConfig.MutablePQConfig()->SetMetaCacheRefreshIntervalMilliSeconds(30000);
+ appConfig.MutablePQConfig()->SetMetaCacheRefreshIntervalMilliSeconds(30000);
appConfig.MutablePQConfig()->MutableQuotingConfig()->SetEnableQuoting(true);
appConfig.MutablePQConfig()->MutableQuotingConfig()->SetQuotaWaitDurationMs(300);
appConfig.MutablePQConfig()->MutableQuotingConfig()->SetPartitionReadQuotaIsTwiceWriteQuota(true);
- appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetEnabled(true);
- appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetFlushIntervalSec(1);
+ appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetEnabled(true);
+ appConfig.MutablePQConfig()->MutableBillingMeteringConfig()->SetFlushIntervalSec(1);
appConfig.MutablePQConfig()->AddClientServiceType()->SetName("data-streams");
-
- MeteringFile = MakeHolder<TTempFileHandle>("meteringData.txt");
- appConfig.MutableMeteringConfig()->SetMeteringFilePath(MeteringFile->Name());
+
+ MeteringFile = MakeHolder<TTempFileHandle>("meteringData.txt");
+ appConfig.MutableMeteringConfig()->SetMeteringFilePath(MeteringFile->Name());
if (secure) {
appConfig.MutablePQConfig()->SetRequireCredentialsInNewProtocol(true);
@@ -86,11 +86,11 @@ public:
std::unique_ptr<TDriver> Driver;
std::unique_ptr<NYDS_V1::TDataStreamsClient> DataStreamsClient;
std::unique_ptr<NYDS_V1::TDataStreamsClient> UnauthenticatedClient;
- THolder<TTempFileHandle> MeteringFile;
+ THolder<TTempFileHandle> MeteringFile;
};
-using TInsecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchema, false>;
-using TSecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchemaSecure, true>;
+using TInsecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchema, false>;
+using TSecureDatastreamsTestServer = TDatastreamsTestServer<TKikimrWithGrpcAndRootSchemaSecure, true>;
void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath) {
Sleep(TDuration::Seconds(1));
@@ -133,8 +133,8 @@ void CheckMeteringFile(TTempFileHandle* meteringFile, const TString& streamPath)
Y_UNIT_TEST_SUITE(DataStreams) {
- Y_UNIT_TEST(TestControlPlaneAndMeteringData) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestControlPlaneAndMeteringData) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
// Trying to delete stream that doesn't exist yet
{
@@ -206,7 +206,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->ListStreams().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->ListStreams().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().stream_names().size(), 1);
@@ -245,8 +245,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
CheckMeteringFile(testServer.MeteringFile.Get(), "/Root/" + streamName);
}
- Y_UNIT_TEST(TestCreateExistingStream) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestCreateExistingStream) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
@@ -264,8 +264,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
- Y_UNIT_TEST(TestStreamPagination) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestStreamPagination) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
for (ui32 folderIdx = 0; folderIdx < 4; folderIdx++) {
for (ui32 streamIdx = 0; streamIdx < 5; streamIdx++) {
@@ -389,8 +389,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestUpdateStream) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestUpdateStream) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
@@ -419,8 +419,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestStreamRetention) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestStreamRetention) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
@@ -463,8 +463,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
- Y_UNIT_TEST(TestShardPagination) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestShardPagination) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(9)).ExtractValueSync();
@@ -516,26 +516,26 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
-
-#define SET_YDS_LOCALS \
- auto& kikimr = testServer.KikimrServer->Server_; \
- Y_UNUSED(kikimr); \
- auto& driver = testServer.Driver; \
- Y_UNUSED(driver); \
-
-
- Y_UNIT_TEST(TestPutRecordsWithRead) {
- TInsecureDatastreamsTestServer testServer;
+
+#define SET_YDS_LOCALS \
+ auto& kikimr = testServer.KikimrServer->Server_; \
+ Y_UNUSED(kikimr); \
+ auto& driver = testServer.Driver; \
+ Y_UNUSED(driver); \
+
+
+ Y_UNIT_TEST(TestPutRecordsWithRead) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- SET_YDS_LOCALS;
+ SET_YDS_LOCALS;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
- kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
- kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG);
+ kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
+ kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG);
NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
@@ -559,7 +559,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
- NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
+ NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
{
auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
@@ -601,20 +601,20 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(readCount, 31);
}
- Y_UNIT_TEST(TestPutRecordsCornerCases) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestPutRecordsCornerCases) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
const TString streamPath = "/Root/" + streamName;
- SET_YDS_LOCALS;
+ SET_YDS_LOCALS;
{
- auto result = testServer.DataStreamsClient->CreateStream(
+ auto result = testServer.DataStreamsClient->CreateStream(
streamName,
NYDS_V1::TCreateStreamSettings().ShardCount(5)
).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
- kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
+ kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
// Test for too long partition key
@@ -679,8 +679,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
- kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_INFO);
- kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PERSQUEUE, NLog::EPriority::PRI_INFO);
+ kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_INFO);
+ kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PERSQUEUE, NLog::EPriority::PRI_INFO);
{
std::vector<NYDS_V1::TDataRecord> records;
@@ -722,7 +722,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
- NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
+ NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
{
NYdb::NPersQueue::TAddReadRuleSettings addReadRuleSettings;
@@ -756,10 +756,10 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(readCount, 14);
}
- Y_UNIT_TEST(TestPutRecords) {
- TSecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestPutRecords) {
+ TSecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- SET_YDS_LOCALS;
+ SET_YDS_LOCALS;
const TString streamPath = "/Root/" + streamName;
{
auto result = testServer.DataStreamsClient->CreateStream(streamPath,
@@ -769,14 +769,14 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
- NYdb::NScheme::TSchemeClient schemeClient(*driver);
+ NYdb::NScheme::TSchemeClient schemeClient(*driver);
{
std::vector<NYDS_V1::TDataRecord> records;
for (ui32 i = 1; i <= 30; ++i) {
TString data = Sprintf("%04u", i);
records.push_back({data, data, ""});
}
- auto result = client.PutRecords(streamPath, records).ExtractValueSync();
+ auto result = client.PutRecords(streamPath, records).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
if (result.GetStatus() != EStatus::SUCCESS) {
result.GetIssues().PrintTo(Cerr);
@@ -790,7 +790,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT(result.IsSuccess());
}
- result = client.PutRecords(streamPath, records).ExtractValueSync();
+ result = client.PutRecords(streamPath, records).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
Cerr << "PutRecordsResponse = " << result.GetResult().DebugString() << Endl;
@@ -799,7 +799,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().encryption_type(), YDS_V1::EncryptionType::NONE);
TString dataStr = "9876543210";
- auto putRecordResult = client.PutRecord(streamPath, {dataStr, dataStr, ""}).ExtractValueSync();
+ auto putRecordResult = client.PutRecord(streamPath, {dataStr, dataStr, ""}).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS);
Cerr << "PutRecord response = " << putRecordResult.GetResult().DebugString() << Endl;
@@ -811,18 +811,18 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestPutEmptyMessage) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestPutEmptyMessage) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- SET_YDS_LOCALS;
+ SET_YDS_LOCALS;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
}
- kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
- kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG);
+ kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NLog::EPriority::PRI_DEBUG);
+ kikimr->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NLog::EPriority::PRI_DEBUG);
NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
@@ -834,7 +834,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(putRecordResult.GetResult().sequence_number(), "0");
- NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
+ NYdb::NPersQueue::TPersQueueClient pqClient(*driver);
{
auto result = testServer.DataStreamsClient->RegisterStreamConsumer(streamName, "user1", NYDS_V1::TRegisterStreamConsumerSettings()).ExtractValueSync();
@@ -873,8 +873,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestListStreamConsumers) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestListStreamConsumers) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
// Create stream -> OK
{
@@ -1070,8 +1070,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestGetShardIterator) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestGetShardIterator) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName, \
@@ -1179,8 +1179,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestGetRecordsStreamWithSingleShard) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestGetRecordsStreamWithSingleShard) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
@@ -1213,7 +1213,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
@@ -1228,7 +1228,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount);
@@ -1245,7 +1245,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 2);
@@ -1262,7 +1262,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount - 3);
@@ -1286,7 +1286,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), recordsCount);
@@ -1294,10 +1294,10 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
- Y_UNIT_TEST(TestGetRecordsWithoutPermission) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestGetRecordsWithoutPermission) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
- SET_YDS_LOCALS;
+ SET_YDS_LOCALS;
{
auto result = testServer.DataStreamsClient->CreateStream("/Root/" + streamName, NYDS_V1::TCreateStreamSettings().ShardCount(1)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
@@ -1316,7 +1316,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
NYDS_V1::TDataStreamsClient client(*driver, TCommonClientSettings().AuthToken("user2@builtin"));
- NYdb::NScheme::TSchemeClient schemeClient(*driver);
+ NYdb::NScheme::TSchemeClient schemeClient(*driver);
TString shardIterator;
{
@@ -1365,8 +1365,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneBySeqNo) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneBySeqNo) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
@@ -1410,7 +1410,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator,
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator,
NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -1423,8 +1423,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneByTS) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestGetRecords1MBMessagesOneByOneByTS) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
@@ -1469,7 +1469,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator,
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator,
NYDS_V1::TGetRecordsSettings().Limit(1)).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
@@ -1480,8 +1480,8 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
- Y_UNIT_TEST(TestGetRecordsStreamWithMultipleShards) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestGetRecordsStreamWithMultipleShards) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName, NYDS_V1::TCreateStreamSettings().ShardCount(5)).ExtractValueSync();
@@ -1514,7 +1514,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
@@ -1529,7 +1529,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
@@ -1544,7 +1544,7 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
@@ -1559,15 +1559,15 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
{
- auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
+ auto result = testServer.DataStreamsClient->GetRecords(shardIterator).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS);
UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records().size(), 0);
}
}
- Y_UNIT_TEST(TestListShards1Shard) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestListShards1Shard) {
+ TInsecureDatastreamsTestServer testServer;
const TString streamName = TStringBuilder() << "stream_" << Y_UNIT_TEST_NAME;
{
auto result = testServer.DataStreamsClient->CreateStream(streamName,
@@ -1626,70 +1626,70 @@ Y_UNIT_TEST_SUITE(DataStreams) {
}
}
- Y_UNIT_TEST(TestUnsupported) {
- TInsecureDatastreamsTestServer testServer;
+ Y_UNIT_TEST(TestUnsupported) {
+ TInsecureDatastreamsTestServer testServer;
{
- auto result = testServer.DataStreamsClient->DescribeLimits().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DescribeLimits().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->DescribeStreamConsumer().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DescribeStreamConsumer().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->AddTagsToStream().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->AddTagsToStream().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->DisableEnhancedMonitoring().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->DisableEnhancedMonitoring().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->EnableEnhancedMonitoring().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->EnableEnhancedMonitoring().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->ListTagsForStream().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->ListTagsForStream().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->MergeShards().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->MergeShards().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->RemoveTagsFromStream().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->RemoveTagsFromStream().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->SplitShard().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->SplitShard().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->StartStreamEncryption().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->StartStreamEncryption().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
{
- auto result = testServer.DataStreamsClient->StopStreamEncryption().ExtractValueSync();
+ auto result = testServer.DataStreamsClient->StopStreamEncryption().ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false);
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED);
}
diff --git a/ydb/services/datastreams/put_records_actor.h b/ydb/services/datastreams/put_records_actor.h
index 5fde6ee96f..8f06a99157 100644
--- a/ydb/services/datastreams/put_records_actor.h
+++ b/ydb/services/datastreams/put_records_actor.h
@@ -13,12 +13,12 @@
#include <library/cpp/digest/md5/md5.h>
-#define PUT_UNIT_SIZE 40960u // 40Kb
+#define PUT_UNIT_SIZE 40960u // 40Kb
namespace NKikimr::NDataStreams::V1 {
-
+
struct TPutRecordsItem {
TString Data;
TString Key;
@@ -98,7 +98,7 @@ namespace NKikimr::NDataStreams::V1 {
request.MutablePartitionRequest()->SetPartition(Partition);
request.MutablePartitionRequest()->SetIsDirectWrite(true);
ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient());
- ui64 totalSize = 0;
+ ui64 totalSize = 0;
for (const auto& item : DataToWrite) {
auto w = request.MutablePartitionRequest()->AddCmdWrite();
w->SetData(GetSerializedData(item));
@@ -108,12 +108,12 @@ namespace NKikimr::NDataStreams::V1 {
w->SetCreateTimeMS(TInstant::Now().MilliSeconds());
w->SetUncompressedSize(item.Data.size());
w->SetExternalOperation(true);
- totalSize += (item.Data.size() + item.Key.size() + item.ExplicitHash.size());
+ totalSize += (item.Data.size() + item.Key.size() + item.ExplicitHash.size());
}
- ui64 putUnitsCount = totalSize / PUT_UNIT_SIZE;
- if (totalSize % PUT_UNIT_SIZE != 0)
- putUnitsCount++;
- request.MutablePartitionRequest()->SetPutUnitsSize(putUnitsCount);
+ ui64 putUnitsCount = totalSize / PUT_UNIT_SIZE;
+ if (totalSize % PUT_UNIT_SIZE != 0)
+ putUnitsCount++;
+ request.MutablePartitionRequest()->SetPutUnitsSize(putUnitsCount);
TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest);
req->Record.Swap(&request);
diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp
index 38aa8f8d0a..f48e398488 100644
--- a/ydb/services/lib/actors/pq_schema_actor.cpp
+++ b/ydb/services/lib/actors/pq_schema_actor.cpp
@@ -11,8 +11,8 @@
namespace NKikimr::NGRpcProxy::V1 {
-#define DEFAULT_PARTITION_SPEED 1048576 // 1Mb
-
+#define DEFAULT_PARTITION_SPEED 1048576 // 1Mb
+
constexpr i32 MAX_READ_RULES_COUNT = 3000;
constexpr TStringBuf GRPCS_ENDPOINT_PREFIX = "grpcs://";
@@ -337,23 +337,23 @@ namespace NKikimr::NGRpcProxy::V1 {
}
if (local) {
- auto partSpeed = settings.max_partition_write_speed();
- if (partSpeed < 0) {
- error = TStringBuilder() << "max_partition_write_speed can't be negative, provided " << partSpeed;
+ auto partSpeed = settings.max_partition_write_speed();
+ if (partSpeed < 0) {
+ error = TStringBuilder() << "max_partition_write_speed can't be negative, provided " << partSpeed;
return Ydb::StatusIds::BAD_REQUEST;
- } else if (partSpeed == 0) {
- partSpeed = DEFAULT_PARTITION_SPEED;
+ } else if (partSpeed == 0) {
+ partSpeed = DEFAULT_PARTITION_SPEED;
}
- partConfig->SetWriteSpeedInBytesPerSecond(partSpeed);
+ partConfig->SetWriteSpeedInBytesPerSecond(partSpeed);
- const auto& burstSpeed = settings.max_partition_write_burst();
- if (burstSpeed < 0) {
- error = TStringBuilder() << "max_partition_write_burst can't be negative, provided " << burstSpeed;
+ const auto& burstSpeed = settings.max_partition_write_burst();
+ if (burstSpeed < 0) {
+ error = TStringBuilder() << "max_partition_write_burst can't be negative, provided " << burstSpeed;
return Ydb::StatusIds::BAD_REQUEST;
- } else if (burstSpeed == 0) {
- partConfig->SetBurstSize(partSpeed);
- } else {
- partConfig->SetBurstSize(burstSpeed);
+ } else if (burstSpeed == 0) {
+ partConfig->SetBurstSize(partSpeed);
+ } else {
+ partConfig->SetBurstSize(burstSpeed);
}
}
diff --git a/ydb/services/lib/actors/pq_schema_actor.h b/ydb/services/lib/actors/pq_schema_actor.h
index 6d77b205ee..1be2e7fa24 100644
--- a/ydb/services/lib/actors/pq_schema_actor.h
+++ b/ydb/services/lib/actors/pq_schema_actor.h
@@ -58,7 +58,7 @@ namespace NKikimr::NGRpcProxy::V1 {
{
}
- void PrepareTopicPath(const NActors::TActorContext &ctx) { // ToDo !!
+ void PrepareTopicPath(const NActors::TActorContext &ctx) { // ToDo !!
TopicPath = NPersQueue::GetFullTopicPath(ctx, this->Request_->GetDatabaseName(), TopicPath);
}
diff --git a/ydb/services/lib/actors/type_definitions.h b/ydb/services/lib/actors/type_definitions.h
index 5f77f3f843..127e7d987b 100644
--- a/ydb/services/lib/actors/type_definitions.h
+++ b/ydb/services/lib/actors/type_definitions.h
@@ -1,39 +1,39 @@
-#pragma once
-
-#include <ydb/library/persqueue/topic_parser/topic_parser.h>
-#include <library/cpp/actors/core/actor.h>
-#include <library/cpp/actors/core/event_local.h>
-
-
-namespace NKikimr::NGRpcProxy {
-
- struct TTopicHolder {
- ui64 TabletID;
- TActorId PipeClient;
- bool ACLRequestInfly;
- TString CloudId;
- TString DbId;
- TString FolderId;
- NPersQueue::TConverterPtr TopicNameConverter;
-
- TVector<ui32> Groups;
- TMap<ui64, ui64> Partitions;
-
- TTopicHolder()
- : TabletID(0)
- , PipeClient()
- , ACLRequestInfly(false)
- {}
- };
-
- struct TTopicInitInfo {
- NPersQueue::TConverterPtr TopicNameConverter;
- ui64 TabletID;
- TString CloudId;
- TString DbId;
- TString FolderId;
- };
-
- using TTopicTabletsPairs = TVector<TTopicInitInfo>;
-
-} // namespace NKikimr::NGRpcProxy
+#pragma once
+
+#include <ydb/library/persqueue/topic_parser/topic_parser.h>
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/event_local.h>
+
+
+namespace NKikimr::NGRpcProxy {
+
+ struct TTopicHolder {
+ ui64 TabletID;
+ TActorId PipeClient;
+ bool ACLRequestInfly;
+ TString CloudId;
+ TString DbId;
+ TString FolderId;
+ NPersQueue::TConverterPtr TopicNameConverter;
+
+ TVector<ui32> Groups;
+ TMap<ui64, ui64> Partitions;
+
+ TTopicHolder()
+ : TabletID(0)
+ , PipeClient()
+ , ACLRequestInfly(false)
+ {}
+ };
+
+ struct TTopicInitInfo {
+ NPersQueue::TConverterPtr TopicNameConverter;
+ ui64 TabletID;
+ TString CloudId;
+ TString DbId;
+ TString FolderId;
+ };
+
+ using TTopicTabletsPairs = TVector<TTopicInitInfo>;
+
+} // namespace NKikimr::NGRpcProxy
diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp
index 0dc27831f3..086d162443 100644
--- a/ydb/services/persqueue_cluster_discovery/grpc_service.cpp
+++ b/ydb/services/persqueue_cluster_discovery/grpc_service.cpp
@@ -13,18 +13,18 @@
namespace NKikimr::NGRpcService {
-TGRpcPQClusterDiscoveryService::TGRpcPQClusterDiscoveryService(
- NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id,
- const TMaybe<ui64>& requestsInflightLimit
-)
+TGRpcPQClusterDiscoveryService::TGRpcPQClusterDiscoveryService(
+ NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters, NActors::TActorId id,
+ const TMaybe<ui64>& requestsInflightLimit
+)
: ActorSystem_(system)
, Counters_(counters)
, GRpcRequestProxyId_(id)
-{
- if (requestsInflightLimit.Defined()) {
- Limiter = MakeHolder<NGrpc::TGlobalLimiter>(requestsInflightLimit.GetRef());
- }
-}
+{
+ if (requestsInflightLimit.Defined()) {
+ Limiter = MakeHolder<NGrpc::TGlobalLimiter>(requestsInflightLimit.GetRef());
+ }
+}
void TGRpcPQClusterDiscoveryService::InitService(grpc::ServerCompletionQueue *cq, NGrpc::TLoggerPtr logger) {
CQ_ = cq;
@@ -42,16 +42,16 @@ void TGRpcPQClusterDiscoveryService::SetGlobalLimiterHandle(NGrpc::TGlobalLimite
}
bool TGRpcPQClusterDiscoveryService::IncRequest() {
- if (Limiter) {
- return Limiter->Inc();
- }
- return true;
+ if (Limiter) {
+ return Limiter->Inc();
+ }
+ return true;
}
void TGRpcPQClusterDiscoveryService::DecRequest() {
- if (Limiter) {
- Limiter->Dec();
- }
+ if (Limiter) {
+ Limiter->Dec();
+ }
}
void TGRpcPQClusterDiscoveryService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
diff --git a/ydb/services/persqueue_cluster_discovery/grpc_service.h b/ydb/services/persqueue_cluster_discovery/grpc_service.h
index 6909f05bd6..7bc17ba2f5 100644
--- a/ydb/services/persqueue_cluster_discovery/grpc_service.h
+++ b/ydb/services/persqueue_cluster_discovery/grpc_service.h
@@ -9,8 +9,8 @@ namespace NKikimr::NGRpcService {
class TGRpcPQClusterDiscoveryService
: public NGrpc::TGrpcServiceBase<Ydb::PersQueue::V1::ClusterDiscoveryService> {
public:
- TGRpcPQClusterDiscoveryService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
- NActors::TActorId id, const TMaybe<ui64>& requestsInflightLimit = Nothing());
+ TGRpcPQClusterDiscoveryService(NActors::TActorSystem* system, TIntrusivePtr<NMonitoring::TDynamicCounters> counters,
+ NActors::TActorId id, const TMaybe<ui64>& requestsInflightLimit = Nothing());
void InitService(grpc::ServerCompletionQueue* cq, NGrpc::TLoggerPtr logger) override;
void SetGlobalLimiterHandle(NGrpc::TGlobalLimiter* limiter) override;
@@ -27,7 +27,7 @@ private:
TIntrusivePtr<NMonitoring::TDynamicCounters> Counters_;
NActors::TActorId GRpcRequestProxyId_;
- THolder<NGrpc::TGlobalLimiter> Limiter;
+ THolder<NGrpc::TGlobalLimiter> Limiter;
};
}
diff --git a/ydb/services/persqueue_v1/grpc_pq_actor.h b/ydb/services/persqueue_v1/grpc_pq_actor.h
index 87c0040147..65c8aca1db 100644
--- a/ydb/services/persqueue_v1/grpc_pq_actor.h
+++ b/ydb/services/persqueue_v1/grpc_pq_actor.h
@@ -76,13 +76,13 @@ static inline bool InternalErrorCode(PersQueue::ErrorCode::ErrorCode errorCode)
}
struct TPartitionId {
- NPersQueue::TConverterPtr TopicConverter;
+ NPersQueue::TConverterPtr TopicConverter;
ui64 Partition;
ui64 AssignId;
bool operator < (const TPartitionId& rhs) const {
- return std::make_tuple(AssignId, Partition, TopicConverter->GetClientsideName()) <
- std::make_tuple(rhs.AssignId, rhs.Partition, rhs.TopicConverter->GetClientsideName());
+ return std::make_tuple(AssignId, Partition, TopicConverter->GetClientsideName()) <
+ std::make_tuple(rhs.AssignId, rhs.Partition, rhs.TopicConverter->GetClientsideName());
}
};
@@ -141,11 +141,11 @@ struct TEvPQProxy {
struct TEvAuthResultOk : public NActors::TEventLocal<TEvAuthResultOk, EvAuthResultOk> {
- TEvAuthResultOk(const TTopicTabletsPairs&& topicAndTablets)
+ TEvAuthResultOk(const TTopicTabletsPairs&& topicAndTablets)
: TopicAndTablets(std::move(topicAndTablets))
{ }
- TTopicTabletsPairs TopicAndTablets;
+ TTopicTabletsPairs TopicAndTablets;
};
struct TEvSessionSetPreferredCluster : public NActors::TEventLocal<TEvSessionSetPreferredCluster, EvSessionSetPreferredCluster> {
@@ -454,14 +454,14 @@ struct TEvPQProxy {
/// WRITE ACTOR
class TWriteSessionActor : public NActors::TActorBootstrapped<TWriteSessionActor> {
using IContext = NGRpcServer::IGRpcStreamingContext<PersQueue::V1::StreamingWriteClientMessage, PersQueue::V1::StreamingWriteServerMessage>;
-using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse;
-using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest;
+using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse;
+using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest;
public:
- TWriteSessionActor(NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie,
- const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
- const NPersQueue::TTopicsListController& topicsController);
+ TWriteSessionActor(NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie,
+ const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
+ const NPersQueue::TTopicsListController& topicsController);
~TWriteSessionActor();
void Bootstrap(const NActors::TActorContext& ctx);
@@ -486,7 +486,7 @@ private:
HFunc(TEvPQProxy::TEvDone, Handle)
HFunc(TEvPersQueue::TEvGetPartitionIdForWriteResponse, Handle)
- HFunc(TEvDescribeTopicsResponse, Handle)
+ HFunc(TEvDescribeTopicsResponse, Handle)
HFunc(NPQ::TEvPartitionWriter::TEvInitResult, Handle);
HFunc(NPQ::TEvPartitionWriter::TEvWriteAccepted, Handle);
@@ -522,7 +522,7 @@ private:
void Handle(TEvPQProxy::TEvUpdateToken::TPtr& ev, const NActors::TActorContext& ctx);
void Handle(TEvPQProxy::TEvDone::TPtr& ev, const NActors::TActorContext& ctx);
void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx);
void LogSession(const TActorContext& ctx);
void DiscoverPartition(const NActors::TActorContext& ctx);
@@ -530,7 +530,7 @@ private:
void RequestNextPartition(const NActors::TActorContext& ctx);
void ProceedPartition(const ui32 partition, const NActors::TActorContext& ctx);
- THolder<NKqp::TEvKqp::TEvQueryRequest> MakeUpdateSourceIdMetadataRequest(const NActors::TActorContext& ctx);
+ THolder<NKqp::TEvKqp::TEvQueryRequest> MakeUpdateSourceIdMetadataRequest(const NActors::TActorContext& ctx);
void InitCheckACL(const TActorContext& ctx);
void Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev, const TActorContext& ctx);
@@ -577,8 +577,8 @@ private:
TString PeerName;
ui64 Cookie;
- NPersQueue::TTopicsListController TopicsController;
- NPersQueue::TConverterPtr TopicConverter;
+ NPersQueue::TTopicsListController TopicsController;
+ NPersQueue::TConverterPtr TopicConverter;
ui32 Partition;
ui32 PreferedPartition;
// 'SourceId' is called 'MessageGroupId' since gRPC data plane API v1
@@ -673,15 +673,15 @@ private:
class TReadInitAndAuthActor : public NActors::TActorBootstrapped<TReadInitAndAuthActor> {
-using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse;
-using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest;
-
+using TEvDescribeTopicsResponse = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsResponse;
+using TEvDescribeTopicsRequest = NMsgBusProxy::NPqMetaCacheV2::TEvPqNewMetaCache::TEvDescribeTopicsRequest;
+
public:
- TReadInitAndAuthActor(const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie,
- const TString& session, const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token,
- const NPersQueue::TTopicsToConverter& topics);
-
+ TReadInitAndAuthActor(const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie,
+ const TString& session, const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token,
+ const NPersQueue::TTopicsToConverter& topics);
+
~TReadInitAndAuthActor();
void Bootstrap(const NActors::TActorContext& ctx);
@@ -691,11 +691,11 @@ public:
private:
- STRICT_STFUNC(StateFunc,
- HFunc(TEvDescribeTopicsResponse, HandleTopicsDescribeResponse)
- HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleClientSchemeCacheResponse)
- HFunc(NActors::TEvents::TEvPoisonPill, HandlePoison)
- );
+ STRICT_STFUNC(StateFunc,
+ HFunc(TEvDescribeTopicsResponse, HandleTopicsDescribeResponse)
+ HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleClientSchemeCacheResponse)
+ HFunc(NActors::TEvents::TEvPoisonPill, HandlePoison)
+ );
void HandlePoison(NActors::TEvents::TEvPoisonPill::TPtr&, const NActors::TActorContext& ctx) {
Die(ctx);
@@ -704,27 +704,27 @@ private:
void CloseSession(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode code, const TActorContext& ctx);
- bool ProcessTopicSchemeCacheResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry,
- THashMap<TString, TTopicHolder>::iterator topicsIter, const TActorContext& ctx);
- void HandleClientSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
+ bool ProcessTopicSchemeCacheResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry,
+ THashMap<TString, TTopicHolder>::iterator topicsIter, const TActorContext& ctx);
+ void HandleClientSchemeCacheResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx);
void SendCacheNavigateRequest(const TActorContext& ctx, const TString& path);
- void HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const NActors::TActorContext& ctx);
+ void HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const NActors::TActorContext& ctx);
void FinishInitialization(const NActors::TActorContext& ctx);
- bool CheckTopicACL(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx);
+ bool CheckTopicACL(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx);
void CheckClientACL(const TActorContext& ctx);
- bool CheckACLPermissionsForNavigate(const TIntrusivePtr<TSecurityObject>& secObject,
- const TString& path, NACLib::EAccessRights rights,
- const TString& errorTextWhenAccessDenied,
- const TActorContext& ctx);
+ bool CheckACLPermissionsForNavigate(const TIntrusivePtr<TSecurityObject>& secObject,
+ const TString& path, NACLib::EAccessRights rights,
+ const TString& errorTextWhenAccessDenied,
+ const TActorContext& ctx);
private:
const TActorId ParentId;
const ui64 Cookie;
const TString Session;
- const TActorId MetaCacheId;
+ const TActorId MetaCacheId;
const TActorId NewSchemeCache;
const TString ClientId;
@@ -735,17 +735,17 @@ private:
THashMap<TString, TTopicHolder> Topics; // topic -> info
TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
- bool DoCheckACL;
+ bool DoCheckACL;
};
class TReadSessionActor : public TActorBootstrapped<TReadSessionActor> {
using IContext = NGRpcServer::IGRpcStreamingContext<PersQueue::V1::MigrationStreamingReadClientMessage, PersQueue::V1::MigrationStreamingReadServerMessage>;
public:
- TReadSessionActor(NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie,
- const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
- const NPersQueue::TTopicsListController& topicsHandler);
+ TReadSessionActor(NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie,
+ const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
+ const NPersQueue::TTopicsListController& topicsHandler);
~TReadSessionActor();
void Bootstrap(const NActors::TActorContext& ctx);
@@ -961,7 +961,7 @@ private:
THashMap<ui64, TPartitionActorInfo> Partitions; //assignId -> info
THashMap<TString, TTopicHolder> Topics; // topic -> info
- THashMap<TString, NPersQueue::TConverterPtr> FullPathToConverter; // PrimaryFullPath -> Converter, for balancer replies matching
+ THashMap<TString, NPersQueue::TConverterPtr> FullPathToConverter; // PrimaryFullPath -> Converter, for balancer replies matching
THashSet<TString> TopicsToResolve;
THashMap<TString, TVector<ui32>> TopicGroups;
THashMap<TString, ui64> ReadFromTimestamp;
@@ -1063,19 +1063,19 @@ private:
NKikimr::NPQ::TMultiCounter SLIBigLatency;
NKikimr::NPQ::TMultiCounter SLIBigReadLatency;
NKikimr::NPQ::TMultiCounter ReadsTotal;
-
- NPersQueue::TTopicsListController TopicsHandler;
+
+ NPersQueue::TTopicsListController TopicsHandler;
};
class TReadInfoActor : public TRpcOperationRequestActor<TReadInfoActor, TEvPQReadInfoRequest> {
using TBase = TRpcOperationRequestActor<TReadInfoActor, TEvPQReadInfoRequest>;
public:
- TReadInfoActor(
- NKikimr::NGRpcService::TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler,
- const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters
- );
+ TReadInfoActor(
+ NKikimr::NGRpcService::TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler,
+ const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters
+ );
~TReadInfoActor();
void Bootstrap(const NActors::TActorContext& ctx);
@@ -1117,12 +1117,12 @@ private:
TActorId AuthInitActor;
- TTopicTabletsPairs TopicAndTablets;
+ TTopicTabletsPairs TopicAndTablets;
TIntrusivePtr<NMonitoring::TDynamicCounters> Counters;
TString ClientId;
- NPersQueue::TTopicsListController TopicsHandler;
+ NPersQueue::TTopicsListController TopicsHandler;
};
diff --git a/ydb/services/persqueue_v1/grpc_pq_read.cpp b/ydb/services/persqueue_v1/grpc_pq_read.cpp
index 705cabf08f..6b7e72fdd0 100644
--- a/ydb/services/persqueue_v1/grpc_pq_read.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_read.cpp
@@ -40,18 +40,18 @@ TPQReadService::TPQReadService(const TActorId& schemeCache, const TActorId& newS
void TPQReadService::Bootstrap(const TActorContext& ctx) {
- HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - proper condition
- if (HaveClusters) {
+ HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - proper condition
+ if (HaveClusters) {
ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(),
new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe);
}
ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
- TopicConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>(
- AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot()
- );
- TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>(
- TopicConverterFactory, HaveClusters, Clusters, LocalCluster
- );
+ TopicConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>(
+ AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot()
+ );
+ TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>(
+ TopicConverterFactory, HaveClusters, Clusters, LocalCluster
+ );
Become(&TThis::StateFunc);
}
@@ -89,7 +89,7 @@ void TPQReadService::Handle(NPQ::NClusterTracker::TEvClusterTracker::TEvClusters
for (size_t i = 0; i < clusters.size(); ++i) {
Clusters[i] = clusters[i].Name;
}
- TopicsHandler->UpdateClusters(Clusters, LocalCluster);
+ TopicsHandler->UpdateClusters(Clusters, LocalCluster);
}
@@ -122,8 +122,8 @@ void TPQReadService::Handle(NKikimr::NGRpcService::TEvStreamPQReadRequest::TPtr&
ev->Get()->GetStreamCtx()->WriteAndFinish(FillReadResponse("proxy overloaded", PersQueue::ErrorCode::OVERLOAD), grpc::Status::OK); //CANCELLED
return;
}
- if (HaveClusters
- && (Clusters.empty() || LocalCluster.empty())) {
+ if (HaveClusters
+ && (Clusters.empty() || LocalCluster.empty())) {
LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, "new grpc connection failed - cluster is not known yet");
ev->Get()->GetStreamCtx()->Attach(ctx.SelfID);
@@ -137,11 +137,11 @@ void TPQReadService::Handle(NKikimr::NGRpcService::TEvStreamPQReadRequest::TPtr&
auto ip = ev->Get()->GetStreamCtx()->GetPeerName();
- TActorId worker = ctx.Register(new TReadSessionActor(
- ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters,
- DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown",
- *TopicsHandler
- ));
+ TActorId worker = ctx.Register(new TReadSessionActor(
+ ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters,
+ DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown",
+ *TopicsHandler
+ ));
Sessions[cookie] = worker;
}
@@ -158,8 +158,8 @@ void TPQReadService::Handle(NKikimr::NGRpcService::TEvPQReadInfoRequest::TPtr& e
ev->Get()->SendResult(ConvertPersQueueInternalCodeToStatus(PersQueue::ErrorCode::INITIALIZING), FillInfoResponse("cluster initializing", PersQueue::ErrorCode::INITIALIZING)); //CANCELLED
return;
} else {
- //ctx.Register(new TReadInfoActor(ev->Release().Release(), Clusters, LocalCluster, SchemeCache, NewSchemeCache, Counters));
- ctx.Register(new TReadInfoActor(ev->Release().Release(), *TopicsHandler, SchemeCache, NewSchemeCache, Counters));
+ //ctx.Register(new TReadInfoActor(ev->Release().Release(), Clusters, LocalCluster, SchemeCache, NewSchemeCache, Counters));
+ ctx.Register(new TReadInfoActor(ev->Release().Release(), *TopicsHandler, SchemeCache, NewSchemeCache, Counters));
}
}
diff --git a/ydb/services/persqueue_v1/grpc_pq_read.h b/ydb/services/persqueue_v1/grpc_pq_read.h
index 2c685fbb18..558063ed13 100644
--- a/ydb/services/persqueue_v1/grpc_pq_read.h
+++ b/ydb/services/persqueue_v1/grpc_pq_read.h
@@ -74,9 +74,9 @@ private:
TString LocalCluster;
NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null
- std::shared_ptr<NPersQueue::TTopicNamesConverterFactory> TopicConverterFactory;
- std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler;
- bool HaveClusters;
+ std::shared_ptr<NPersQueue::TTopicNamesConverterFactory> TopicConverterFactory;
+ std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler;
+ bool HaveClusters;
};
diff --git a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
index c7e2c11407..654edfcfcd 100644
--- a/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_read_actor.cpp
@@ -71,7 +71,7 @@ static const double LAG_GROW_MULTIPLIER = 1.2; //assume that 20% more data arriv
IOutputStream& operator <<(IOutputStream& out, const TPartitionId& partId) {
- out << "TopicId: " << partId.TopicConverter->GetClientsideName() << ":" << partId.Partition << "(assignId:" << partId.AssignId << ")";
+ out << "TopicId: " << partId.TopicConverter->GetClientsideName() << ":" << partId.Partition << "(assignId:" << partId.AssignId << ")";
return out;
}
@@ -142,8 +142,8 @@ private:
void Handle(TEvPQProxy::TEvReleasePartition::TPtr& ev, const NActors::TActorContext& ctx);
void Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const NActors::TActorContext& ctx);
- void Handle(TEvPQProxy::TEvGetStatus::
- TPtr& ev, const NActors::TActorContext& ctx);
+ void Handle(TEvPQProxy::TEvGetStatus::
+ TPtr& ev, const NActors::TActorContext& ctx);
void Handle(TEvPQProxy::TEvDeadlineExceeded::TPtr& ev, const NActors::TActorContext& ctx);
@@ -236,11 +236,11 @@ private:
};
-TReadSessionActor::TReadSessionActor(
- NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie, const TActorId& schemeCache, const TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
- const NPersQueue::TTopicsListController& topicsHandler
-)
+TReadSessionActor::TReadSessionActor(
+ NKikimr::NGRpcService::TEvStreamPQReadRequest* request, const ui64 cookie, const TActorId& schemeCache, const TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
+ const NPersQueue::TTopicsListController& topicsHandler
+)
: Request(request)
, ClientDC(clientDC ? *clientDC : "other")
, StartTimestamp(TInstant::Now())
@@ -267,7 +267,7 @@ TReadSessionActor::TReadSessionActor(
, BytesInflight_(0)
, RequestedBytes(0)
, ReadsInfly(0)
- , TopicsHandler(topicsHandler)
+ , TopicsHandler(topicsHandler)
{
Y_ASSERT(Request);
++(*GetServiceCounters(Counters, "pqproxy|readSession")->GetCounter("SessionsCreatedTotal", true));
@@ -285,9 +285,9 @@ void TReadSessionActor::Bootstrap(const TActorContext& ctx) {
Die(ctx);
return;
}
- //auto& pqConfig = AppData(ctx)->PQConfig;
- //bool noDcMode = !pqConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - add multicluster mode
- //ConverterFactory = MakeHolder<NPersQueue::TTopicNamesConverterFactory>(noDcMode, pqConfig.GetRoot());
+ //auto& pqConfig = AppData(ctx)->PQConfig;
+ //bool noDcMode = !pqConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration] - add multicluster mode
+ //ConverterFactory = MakeHolder<NPersQueue::TTopicNamesConverterFactory>(noDcMode, pqConfig.GetRoot());
StartTime = ctx.Now();
Become(&TThis::StateFunc);
@@ -317,29 +317,29 @@ void TReadSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActor
ctx.Send(ctx.SelfID, new TEvPQProxy::TEvDone());
return;
}
- auto converterFactory = TopicsHandler.GetConverterFactory();
- auto MakePartitionId = [&](auto& request) {
- auto converter = converterFactory->MakeTopicNameConverter(
- request.topic().path(), request.cluster(), Request->GetDatabaseName().GetOrElse(TString())
- );
- if (!converter->IsValid()) {
- CloseSession(
- TStringBuilder() << "Topic " << request.topic().path() << " not recognized: " << converter->GetReason(),
- PersQueue::ErrorCode::BAD_REQUEST, ctx
- );
- }
- const ui32 partition = request.partition();
- const ui64 assignId = request.assign_id();
- return TPartitionId{converter, partition, assignId};
- };
+ auto converterFactory = TopicsHandler.GetConverterFactory();
+ auto MakePartitionId = [&](auto& request) {
+ auto converter = converterFactory->MakeTopicNameConverter(
+ request.topic().path(), request.cluster(), Request->GetDatabaseName().GetOrElse(TString())
+ );
+ if (!converter->IsValid()) {
+ CloseSession(
+ TStringBuilder() << "Topic " << request.topic().path() << " not recognized: " << converter->GetReason(),
+ PersQueue::ErrorCode::BAD_REQUEST, ctx
+ );
+ }
+ const ui32 partition = request.partition();
+ const ui64 assignId = request.assign_id();
+ return TPartitionId{converter, partition, assignId};
+ };
switch (request.request_case()) {
case MigrationStreamingReadClientMessage::kInitRequest: {
ctx.Send(ctx.SelfID, new TEvPQProxy::TEvReadInit(request, Request->GetStreamCtx()->GetPeerName()));
break;
}
case MigrationStreamingReadClientMessage::kStatus: {
- //const auto& req = request.status();
- ctx.Send(ctx.SelfID, new TEvPQProxy::TEvGetStatus(MakePartitionId(request.status())));
+ //const auto& req = request.status();
+ ctx.Send(ctx.SelfID, new TEvPQProxy::TEvGetStatus(MakePartitionId(request.status())));
if (!Request->GetStreamCtx()->Read()) {
LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " grpc read failed at start");
Die(ctx);
@@ -353,8 +353,8 @@ void TReadSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActor
break;
}
case MigrationStreamingReadClientMessage::kReleased: {
- //const auto& req = request.released();
- ctx.Send(ctx.SelfID, new TEvPQProxy::TEvReleased(MakePartitionId(request.released())));
+ //const auto& req = request.released();
+ ctx.Send(ctx.SelfID, new TEvPQProxy::TEvReleased(MakePartitionId(request.released())));
if (!Request->GetStreamCtx()->Read()) {
LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " grpc read failed at start");
Die(ctx);
@@ -370,7 +370,7 @@ void TReadSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActor
const ui64 commitOffset = req.commit_offset();
const bool verifyReadOffset = req.verify_read_offset();
- ctx.Send(ctx.SelfID, new TEvPQProxy::TEvStartRead(MakePartitionId(request.start_read()), readOffset, commitOffset, verifyReadOffset));
+ ctx.Send(ctx.SelfID, new TEvPQProxy::TEvStartRead(MakePartitionId(request.start_read()), readOffset, commitOffset, verifyReadOffset));
if (!Request->GetStreamCtx()->Read()) {
LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " grpc read failed at start");
Die(ctx);
@@ -449,7 +449,7 @@ void TReadSessionActor::Die(const TActorContext& ctx) {
ctx.Send(p.second.Actor, new TEvents::TEvPoisonPill());
if (!p.second.Released) {
- auto it = TopicCounters.find(p.second.Partition.TopicConverter->GetClientsideName());
+ auto it = TopicCounters.find(p.second.Partition.TopicConverter->GetClientsideName());
Y_VERIFY(it != TopicCounters.end());
it->second.PartitionsInfly.Dec();
it->second.PartitionsReleased.Inc();
@@ -600,8 +600,8 @@ void TReadSessionActor::DropPartition(THashMap<ui64, TPartitionActorInfo>::itera
bool res = ActualPartitionActors.erase(it->second.Actor);
Y_VERIFY(res);
- if (--NumPartitionsFromTopic[it->second.Partition.TopicConverter->GetClientsideName()] == 0) {
- bool res = TopicCounters.erase(it->second.Partition.TopicConverter->GetClientsideName());
+ if (--NumPartitionsFromTopic[it->second.Partition.TopicConverter->GetClientsideName()] == 0) {
+ bool res = TopicCounters.erase(it->second.Partition.TopicConverter->GetClientsideName());
Y_VERIFY(res);
}
@@ -662,7 +662,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadSessionStatus::TPtr& ev, const
THolder<TEvPQProxy::TEvReadSessionStatusResponse> result(new TEvPQProxy::TEvReadSessionStatusResponse());
for (auto& p : Partitions) {
auto part = result->Record.AddPartition();
- part->SetTopic(p.second.Partition.TopicConverter->GetPrimaryPath());
+ part->SetTopic(p.second.Partition.TopicConverter->GetPrimaryPath());
part->SetPartition(p.second.Partition.Partition);
part->SetAssignId(p.second.Partition.AssignId);
for (auto& c : p.second.NextCommits) {
@@ -730,11 +730,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo
ReadOnlyLocal = init.read_only_original();
for (const auto& topic : init.topics_read_settings()) {
- auto converter = TopicsHandler.GetConverterFactory()->MakeTopicNameConverter(
- topic.topic(), TString(), Request->GetDatabaseName().GetOrElse(TString())
- );
- const auto topicName = converter->GetModernName();
- if (topicName.empty()) {
+ auto converter = TopicsHandler.GetConverterFactory()->MakeTopicNameConverter(
+ topic.topic(), TString(), Request->GetDatabaseName().GetOrElse(TString())
+ );
+ const auto topicName = converter->GetModernName();
+ if (topicName.empty()) {
CloseSession("empty topic in init request", PersQueue::ErrorCode::BAD_REQUEST, ctx);
return;
}
@@ -742,7 +742,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo
CloseSession("start_from_written_at_ms must be nonnegative number", PersQueue::ErrorCode::BAD_REQUEST, ctx);
return;
}
- TopicsToResolve.insert(topicName);
+ TopicsToResolve.insert(topicName);
for (i64 pg : topic.partition_group_ids()) {
if (pg < 0) {
CloseSession("partition group id must be nonnegative number", PersQueue::ErrorCode::BAD_REQUEST, ctx);
@@ -752,9 +752,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo
CloseSession(TStringBuilder() << "partition group id " << pg << " is too big for partition group id", PersQueue::ErrorCode::BAD_REQUEST, ctx);
return;
}
- TopicGroups[topicName].push_back(static_cast<ui32>(pg));
+ TopicGroups[topicName].push_back(static_cast<ui32>(pg));
}
- ReadFromTimestamp[topicName] = topic.start_from_written_at_ms();
+ ReadFromTimestamp[topicName] = topic.start_from_written_at_ms();
}
LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " init: " << event->Request << " from " << PeerName);
@@ -770,10 +770,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo
Auth = *(Request->GetYdbToken());
Token = new NACLib::TUserToken(Request->GetInternalToken());
}
- AuthInitActor = ctx.Register(new TReadInitAndAuthActor(
- ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token,
- TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString()))
- ));
+ AuthInitActor = ctx.Register(new TReadInitAndAuthActor(
+ ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token,
+ TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString()))
+ ));
auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
@@ -811,7 +811,7 @@ void TReadSessionActor::RegisterSessions(const TActorContext& ctx) {
for (auto& t : Topics) {
auto& topic = t.first;
RegisterSession(t.second.PipeClient, topic, t.second.Groups, ctx);
- NumPartitionsFromTopic[t.second.TopicNameConverter->GetClientsideName()] = 0;
+ NumPartitionsFromTopic[t.second.TopicNameConverter->GetClientsideName()] = 0;
}
}
@@ -887,11 +887,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct
LastACLCheckTimestamp = ctx.Now();
- LOG_INFO_S(
- ctx,
- NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone
- );
+ LOG_INFO_S(
+ ctx,
+ NKikimrServices::PQ_READ_PROXY,
+ PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone
+ );
AuthInitActor = TActorId();
@@ -935,11 +935,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct
Y_VERIFY(!BalancersInitStarted);
BalancersInitStarted = true;
- for (auto& t : ev->Get()->TopicAndTablets) { // ToDo - return something from Init and Auth Actor (Full Path - ?)
- auto& topicHolder = Topics[t.TopicNameConverter->GetClientsideName()];
- topicHolder.TabletID = t.TabletID;
- topicHolder.TopicNameConverter = t.TopicNameConverter;
- FullPathToConverter[t.TopicNameConverter->GetPrimaryPath()] = t.TopicNameConverter;
+ for (auto& t : ev->Get()->TopicAndTablets) { // ToDo - return something from Init and Auth Actor (Full Path - ?)
+ auto& topicHolder = Topics[t.TopicNameConverter->GetClientsideName()];
+ topicHolder.TabletID = t.TabletID;
+ topicHolder.TopicNameConverter = t.TopicNameConverter;
+ FullPathToConverter[t.TopicNameConverter->GetPrimaryPath()] = t.TopicNameConverter;
}
for (auto& t : Topics) {
@@ -949,8 +949,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct
clientConfig.RetryPolicy = RetryPolicyForPipes;
t.second.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, t.second.TabletID, clientConfig));
-
- auto it = TopicGroups.find(t.second.TopicNameConverter->GetModernName());
+
+ auto it = TopicGroups.find(t.second.TopicNameConverter->GetModernName());
if (it != TopicGroups.end()) {
t.second.Groups = it->second;
}
@@ -961,12 +961,12 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAct
ctx.Schedule(CHECK_ACL_DELAY, new TEvents::TEvWakeup());
} else {
for (auto& t : ev->Get()->TopicAndTablets) {
- if (Topics.find(t.TopicNameConverter->GetClientsideName()) == Topics.end()) {
- CloseSession(
- TStringBuilder() << "list of topics changed - new topic '"
- << t.TopicNameConverter->GetClientsideName() << "' found",
- PersQueue::ErrorCode::BAD_REQUEST, ctx
- );
+ if (Topics.find(t.TopicNameConverter->GetClientsideName()) == Topics.end()) {
+ CloseSession(
+ TStringBuilder() << "list of topics changed - new topic '"
+ << t.TopicNameConverter->GetClientsideName() << "' found",
+ PersQueue::ErrorCode::BAD_REQUEST, ctx
+ );
return;
}
}
@@ -981,23 +981,23 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T
Y_VERIFY(record.GetClientId() == ClientId);
TActorId pipe = ActorIdFromProto(record.GetPipeClient());
- auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(record.GetPath()));
-
- if (converterIter.IsEnd()) {
- LOG_DEBUG_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " ignored ev lock for path = " << record.GetPath() << ", path not recognized"
- );
- return;
- }
- const auto& topic = converterIter->second->GetClientsideName();
- auto jt = Topics.find(topic); // ToDo - Check
+ auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(record.GetPath()));
+
+ if (converterIter.IsEnd()) {
+ LOG_DEBUG_S(
+ ctx, NKikimrServices::PQ_READ_PROXY,
+ PQ_LOG_PREFIX << " ignored ev lock for path = " << record.GetPath() << ", path not recognized"
+ );
+ return;
+ }
+ const auto& topic = converterIter->second->GetClientsideName();
+ auto jt = Topics.find(topic); // ToDo - Check
if (jt == Topics.end() || pipe != jt->second.PipeClient) { //this is message from old version of pipe
- LOG_ALERT_S(
- ctx, NKikimrServices::PQ_READ_PROXY,
- PQ_LOG_PREFIX << " ignored ev lock for topic = " << topic
- << " path recognized, but topic is unknown, this is unexpected"
- );
+ LOG_ALERT_S(
+ ctx, NKikimrServices::PQ_READ_PROXY,
+ PQ_LOG_PREFIX << " ignored ev lock for topic = " << topic
+ << " path recognized, but topic is unknown, this is unexpected"
+ );
return;
}
@@ -1014,7 +1014,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T
ui64 assignId = NextAssignId++;
BalancerGeneration[assignId] = {record.GetGeneration(), record.GetStep()};
- TPartitionId partitionId{converterIter->second, record.GetPartition(), assignId};
+ TPartitionId partitionId{converterIter->second, record.GetPartition(), assignId};
IActor* partitionActor = new TPartitionActor(ctx.SelfID, ClientId, ClientPath, Cookie, Session, partitionId, record.GetGeneration(),
record.GetStep(), record.GetTabletId(), it->second, CommitsDisabled, ClientDC);
@@ -1054,8 +1054,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const T
MigrationStreamingReadServerMessage result;
result.set_status(Ydb::StatusIds::SUCCESS);
- result.mutable_assigned()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName());
- result.mutable_assigned()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster());
+ result.mutable_assigned()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName());
+ result.mutable_assigned()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster());
result.mutable_assigned()->set_partition(ev->Get()->Partition.Partition);
result.mutable_assigned()->set_assign_id(it->first);
@@ -1083,8 +1083,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const T
MigrationStreamingReadServerMessage result;
result.set_status(Ydb::StatusIds::SUCCESS);
- result.mutable_partition_status()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName());
- result.mutable_partition_status()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster());
+ result.mutable_partition_status()->mutable_topic()->set_path(ev->Get()->Partition.TopicConverter->GetModernName());
+ result.mutable_partition_status()->set_cluster(ev->Get()->Partition.TopicConverter->GetCluster());
result.mutable_partition_status()->set_partition(ev->Get()->Partition.Partition);
result.mutable_partition_status()->set_assign_id(it->first);
@@ -1118,8 +1118,8 @@ void TReadSessionActor::SendReleaseSignalToClient(const THashMap<ui64, TPartitio
MigrationStreamingReadServerMessage result;
result.set_status(Ydb::StatusIds::SUCCESS);
- result.mutable_release()->mutable_topic()->set_path(it->second.Partition.TopicConverter->GetModernName());
- result.mutable_release()->set_cluster(it->second.Partition.TopicConverter->GetCluster());
+ result.mutable_release()->mutable_topic()->set_path(it->second.Partition.TopicConverter->GetModernName());
+ result.mutable_release()->set_cluster(it->second.Partition.TopicConverter->GetCluster());
result.mutable_release()->set_partition(it->second.Partition.Partition);
result.mutable_release()->set_assign_id(it->second.Partition.AssignId);
result.mutable_release()->set_forceful_release(kill);
@@ -1147,14 +1147,14 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons
auto& record = ev->Get()->Record;
Y_VERIFY(record.GetSession() == Session);
Y_VERIFY(record.GetClientId() == ClientId);
- TString topicPath = NPersQueue::NormalizeFullPath(record.GetPath());
+ TString topicPath = NPersQueue::NormalizeFullPath(record.GetPath());
ui32 group = record.HasGroup() ? record.GetGroup() : 0;
- auto pathIter = FullPathToConverter.find(topicPath);
- Y_VERIFY(!pathIter.IsEnd());
- auto it = Topics.find(pathIter->second->GetClientsideName());
- Y_VERIFY(!it.IsEnd());
- auto& converter = it->second.TopicNameConverter;
+ auto pathIter = FullPathToConverter.find(topicPath);
+ Y_VERIFY(!pathIter.IsEnd());
+ auto it = Topics.find(pathIter->second->GetClientsideName());
+ Y_VERIFY(!it.IsEnd());
+ auto& converter = it->second.TopicNameConverter;
TActorId pipe = ActorIdFromProto(record.GetPipeClient());
@@ -1169,10 +1169,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons
auto jt = Partitions.begin();
ui32 i = 0;
for (auto it = Partitions.begin(); it != Partitions.end(); ++it) {
- if (it->second.Partition.TopicConverter->GetPrimaryPath() == converter->GetPrimaryPath()
- && !it->second.Releasing
- && (group == 0 || it->second.Partition.Partition + 1 == group)
- ) {
+ if (it->second.Partition.TopicConverter->GetPrimaryPath() == converter->GetPrimaryPath()
+ && !it->second.Releasing
+ && (group == 0 || it->second.Partition.Partition + 1 == group)
+ ) {
++i;
if (rand() % i == 0) { //will lead to 1/n probability for each of n partitions
actorId = it->second.Actor;
@@ -1183,7 +1183,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons
Y_VERIFY(actorId);
{
- auto it = TopicCounters.find(converter->GetClientsideName());
+ auto it = TopicCounters.find(converter->GetClientsideName());
Y_VERIFY(it != TopicCounters.end());
it->second.PartitionsToBeReleased.Inc();
}
@@ -1218,14 +1218,14 @@ void TReadSessionActor::InformBalancerAboutRelease(const THashMap<ui64, TPartiti
request.Reset(new TEvPersQueue::TEvPartitionReleased);
auto& req = request->Record;
- const auto& converter = it->second.Partition.TopicConverter;
- auto jt = Topics.find(converter->GetClientsideName());
+ const auto& converter = it->second.Partition.TopicConverter;
+ auto jt = Topics.find(converter->GetClientsideName());
Y_VERIFY(jt != Topics.end());
req.SetSession(Session);
ActorIdToProto(jt->second.PipeClient, req.MutablePipeClient());
req.SetClientId(ClientId);
- req.SetTopic(converter->GetPrimaryPath());
+ req.SetTopic(converter->GetPrimaryPath());
req.SetPartition(it->second.Partition.Partition);
LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " released: " << it->second.Partition);
@@ -1300,7 +1300,7 @@ void TReadSessionActor::ReleasePartition(const THashMap<ui64, TPartitionActorInf
bool couldBeReads, const TActorContext& ctx)
{
{
- auto jt = TopicCounters.find(it->second.Partition.TopicConverter->GetClientsideName());
+ auto jt = TopicCounters.find(it->second.Partition.TopicConverter->GetClientsideName());
Y_VERIFY(jt != TopicCounters.end());
jt->second.PartitionsReleased.Inc();
jt->second.PartitionsInfly.Dec();
@@ -1338,7 +1338,7 @@ bool TReadSessionActor::ProcessBalancerDead(const ui64 tablet, const TActorConte
//Drop all partitions from this topic
for (auto it = Partitions.begin(); it != Partitions.end();) {
- if (it->second.Partition.TopicConverter->GetClientsideName() == t.first) { //partition from this topic
+ if (it->second.Partition.TopicConverter->GetClientsideName() == t.first) { //partition from this topic
// kill actor
auto jt = it;
++it;
@@ -1580,7 +1580,7 @@ void TReadSessionActor::ProcessReads(const TActorContext& ctx) {
size -= csize;
Y_VERIFY(csize < Max<i32>());
- auto jt = ReadFromTimestamp.find(it->second.Partition.TopicConverter->GetModernName());
+ auto jt = ReadFromTimestamp.find(it->second.Partition.TopicConverter->GetModernName());
Y_VERIFY(jt != ReadFromTimestamp.end());
ui64 readTimestampMs = Max(ReadTimestampMs, jt->second);
@@ -1659,10 +1659,10 @@ void TReadSessionActor::HandleWakeup(const TActorContext& ctx) {
Y_VERIFY(!AuthInitActor);
LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " checking auth because of timeout");
- AuthInitActor = ctx.Register(new TReadInitAndAuthActor(
- ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token,
- TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString()))
- ));
+ AuthInitActor = ctx.Register(new TReadInitAndAuthActor(
+ ctx, ctx.SelfID, ClientId, Cookie, Session, SchemeCache, NewSchemeCache, Counters, Token,
+ TopicsHandler.GetReadTopicsList(TopicsToResolve, ReadOnlyLocal, Request->GetDatabaseName().GetOrElse(TString()))
+ ));
}
}
@@ -1811,7 +1811,7 @@ void TPartitionActor::CheckRelease(const TActorContext& ctx) {
void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) {
NKikimrClient::TPersQueueRequest request;
- request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath());
+ request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath());
request.MutablePartitionRequest()->SetPartition(Partition.Partition);
request.MutablePartitionRequest()->SetCookie(readId);
@@ -1903,10 +1903,10 @@ void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TAct
bool FillBatchedData(MigrationStreamingReadServerMessage::DataBatch * data, const NKikimrClient::TCmdReadResult& res, const TPartitionId& Partition, ui64 ReadIdToResponse, ui64& ReadOffset, ui64& WTime, ui64 EndOffset, const TActorContext& ctx) {
auto* partitionData = data->add_partition_data();
- partitionData->mutable_topic()->set_path(Partition.TopicConverter->GetModernName());
- partitionData->set_cluster(Partition.TopicConverter->GetCluster());
+ partitionData->mutable_topic()->set_path(Partition.TopicConverter->GetModernName());
+ partitionData->set_cluster(Partition.TopicConverter->GetCluster());
partitionData->set_partition(Partition.Partition);
- partitionData->set_deprecated_topic(Partition.TopicConverter->GetFullLegacyName());
+ partitionData->set_deprecated_topic(Partition.TopicConverter->GetFullLegacyName());
partitionData->mutable_cookie()->set_assign_id(Partition.AssignId);
partitionData->mutable_cookie()->set_partition_cookie(ReadIdToResponse);
@@ -2318,7 +2318,7 @@ void TPartitionActor::InitLockPartition(const TActorContext& ctx) {
NKikimrClient::TPersQueueRequest request;
- request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath());
+ request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath());
request.MutablePartitionRequest()->SetPartition(Partition.Partition);
request.MutablePartitionRequest()->SetCookie(INIT_COOKIE);
@@ -2438,7 +2438,7 @@ void TPartitionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext&
NKikimrClient::TPersQueueRequest request;
- request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath());
+ request.MutablePartitionRequest()->SetTopic(Partition.TopicConverter->GetPrimaryPath());
request.MutablePartitionRequest()->SetPartition(Partition.Partition);
request.MutablePartitionRequest()->SetCookie((ui64)ReadOffset);
@@ -2565,16 +2565,16 @@ void TPartitionActor::HandleWakeup(const TActorContext& ctx) {
}
///////////////// AuthAndInitActor
-TReadInitAndAuthActor::TReadInitAndAuthActor(
- const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie,
- const TString& session, const NActors::TActorId& metaCache, const NActors::TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token,
- const NPersQueue::TTopicsToConverter& topics
-)
+TReadInitAndAuthActor::TReadInitAndAuthActor(
+ const TActorContext& ctx, const TActorId& parentId, const TString& clientId, const ui64 cookie,
+ const TString& session, const NActors::TActorId& metaCache, const NActors::TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters, TIntrusivePtr<NACLib::TUserToken> token,
+ const NPersQueue::TTopicsToConverter& topics
+)
: ParentId(parentId)
, Cookie(cookie)
, Session(session)
- , MetaCacheId(metaCache)
+ , MetaCacheId(metaCache)
, NewSchemeCache(newSchemeCache)
, ClientId(clientId)
, ClientPath(NPersQueue::ConvertOldConsumerName(ClientId, ctx))
@@ -2582,7 +2582,7 @@ TReadInitAndAuthActor::TReadInitAndAuthActor(
, Counters(counters)
{
for (const auto& t : topics) {
- Topics[t.first].TopicNameConverter = t.second;
+ Topics[t.first].TopicNameConverter = t.second;
}
}
@@ -2592,13 +2592,13 @@ TReadInitAndAuthActor::~TReadInitAndAuthActor() = default;
void TReadInitAndAuthActor::Bootstrap(const TActorContext &ctx) {
LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " auth for : " << ClientId);
- Become(&TThis::StateFunc);
- TVector<TString> topicNames;
- for (const auto& topic : Topics) {
- topicNames.emplace_back(topic.second.TopicNameConverter->GetPrimaryPath());
+ Become(&TThis::StateFunc);
+ TVector<TString> topicNames;
+ for (const auto& topic : Topics) {
+ topicNames.emplace_back(topic.second.TopicNameConverter->GetPrimaryPath());
}
- DoCheckACL = AppData(ctx)->PQConfig.GetCheckACL() && Token;
- ctx.Send(MetaCacheId, new TEvDescribeTopicsRequest(topicNames));
+ DoCheckACL = AppData(ctx)->PQConfig.GetCheckACL() && Token;
+ ctx.Send(MetaCacheId, new TEvDescribeTopicsRequest(topicNames));
}
void TReadInitAndAuthActor::Die(const TActorContext& ctx) {
@@ -2625,89 +2625,89 @@ void TReadInitAndAuthActor::SendCacheNavigateRequest(const TActorContext& ctx, c
entry.SyncVersion = true;
entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath;
schemeCacheRequest->ResultSet.emplace_back(entry);
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Send client acl request");
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Send client acl request");
ctx.Send(NewSchemeCache, new TEvTxProxySchemeCache::TEvNavigateKeySet(schemeCacheRequest.Release()));
}
-bool TReadInitAndAuthActor::ProcessTopicSchemeCacheResponse(
- const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, THashMap<TString, TTopicHolder>::iterator topicsIter,
- const TActorContext& ctx
-) {
- auto& pqDescr = entry.PQGroupInfo->Description;
- Y_VERIFY(entry.PQGroupInfo);
- topicsIter->second.TabletID = pqDescr.GetBalancerTabletID();
- return CheckTopicACL(entry, topicsIter->first, ctx);
-}
-
-
-void TReadInitAndAuthActor::HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Handle describe topics response");
- for (const auto& entry : ev->Get()->Result->ResultSet) {
- auto processResult = ProcessMetaCacheTopicResponse(entry);
- auto path = JoinPath(entry.Path);
- auto it = Topics.find(path);
- Y_VERIFY(it != Topics.end());
-
- if (processResult.IsFatal) {
- Topics.erase(it);
- if (Topics.empty()) {
- TStringBuilder reason;
- reason << "Discovery for all topics failed. The last error was: " << processResult.Reason;
- return CloseSession(reason, processResult.ErrorCode, ctx);
- } else {
- continue;
- }
- }
- if (!ProcessTopicSchemeCacheResponse(entry, it, ctx))
- return;
-
- }
- if (Topics.empty()) {
- CloseSession("no topics found", PersQueue::ErrorCode::BAD_REQUEST, ctx);
- return;
- }
- // ToDo[migration] - separate option - ?
- bool doCheckClientAcl = DoCheckACL && !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen();
- if (doCheckClientAcl) {
- CheckClientACL(ctx);
- } else {
- FinishInitialization(ctx);
- }
-}
-
-
-bool TReadInitAndAuthActor::CheckTopicACL(
- const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx
-) {
- auto& pqDescr = entry.PQGroupInfo->Description;
- //ToDo[migration] - proper auth setup
- bool alwaysCheckPermissions = AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen();
- bool reqAuthRead = DoCheckACL && (
- pqDescr.GetPQTabletConfig().GetRequireAuthRead() || alwaysCheckPermissions
- );
-
- if (reqAuthRead && !CheckACLPermissionsForNavigate(
- entry.SecurityObject, topic, NACLib::EAccessRights::SelectRow,
- "No ReadTopic permissions", ctx
- )) {
- return false;
- }
- //ToDo[migration] - proper auth setup
- bool doCheckReadRules = AppData(ctx)->PQConfig.GetCheckACL() && (Token || alwaysCheckPermissions);
- if (doCheckReadRules) {
- bool found = false;
- for (auto& cons : pqDescr.GetPQTabletConfig().GetReadRules() ) {
- if (cons == ClientId) {
- found = true;
- break;
+bool TReadInitAndAuthActor::ProcessTopicSchemeCacheResponse(
+ const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, THashMap<TString, TTopicHolder>::iterator topicsIter,
+ const TActorContext& ctx
+) {
+ auto& pqDescr = entry.PQGroupInfo->Description;
+ Y_VERIFY(entry.PQGroupInfo);
+ topicsIter->second.TabletID = pqDescr.GetBalancerTabletID();
+ return CheckTopicACL(entry, topicsIter->first, ctx);
+}
+
+
+void TReadInitAndAuthActor::HandleTopicsDescribeResponse(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) {
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Handle describe topics response");
+ for (const auto& entry : ev->Get()->Result->ResultSet) {
+ auto processResult = ProcessMetaCacheTopicResponse(entry);
+ auto path = JoinPath(entry.Path);
+ auto it = Topics.find(path);
+ Y_VERIFY(it != Topics.end());
+
+ if (processResult.IsFatal) {
+ Topics.erase(it);
+ if (Topics.empty()) {
+ TStringBuilder reason;
+ reason << "Discovery for all topics failed. The last error was: " << processResult.Reason;
+ return CloseSession(reason, processResult.ErrorCode, ctx);
+ } else {
+ continue;
+ }
+ }
+ if (!ProcessTopicSchemeCacheResponse(entry, it, ctx))
+ return;
+
+ }
+ if (Topics.empty()) {
+ CloseSession("no topics found", PersQueue::ErrorCode::BAD_REQUEST, ctx);
+ return;
+ }
+ // ToDo[migration] - separate option - ?
+ bool doCheckClientAcl = DoCheckACL && !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen();
+ if (doCheckClientAcl) {
+ CheckClientACL(ctx);
+ } else {
+ FinishInitialization(ctx);
+ }
+}
+
+
+bool TReadInitAndAuthActor::CheckTopicACL(
+ const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TString& topic, const TActorContext& ctx
+) {
+ auto& pqDescr = entry.PQGroupInfo->Description;
+ //ToDo[migration] - proper auth setup
+ bool alwaysCheckPermissions = AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen();
+ bool reqAuthRead = DoCheckACL && (
+ pqDescr.GetPQTabletConfig().GetRequireAuthRead() || alwaysCheckPermissions
+ );
+
+ if (reqAuthRead && !CheckACLPermissionsForNavigate(
+ entry.SecurityObject, topic, NACLib::EAccessRights::SelectRow,
+ "No ReadTopic permissions", ctx
+ )) {
+ return false;
+ }
+ //ToDo[migration] - proper auth setup
+ bool doCheckReadRules = AppData(ctx)->PQConfig.GetCheckACL() && (Token || alwaysCheckPermissions);
+ if (doCheckReadRules) {
+ bool found = false;
+ for (auto& cons : pqDescr.GetPQTabletConfig().GetReadRules() ) {
+ if (cons == ClientId) {
+ found = true;
+ break;
}
}
- if (!found) {
- CloseSession(
- TStringBuilder() << "no read rule provided for consumer '" << ClientPath << "' in topic '" << topic << "'",
- PersQueue::ErrorCode::BAD_REQUEST, ctx
- );
+ if (!found) {
+ CloseSession(
+ TStringBuilder() << "no read rule provided for consumer '" << ClientPath << "' in topic '" << topic << "'",
+ PersQueue::ErrorCode::BAD_REQUEST, ctx
+ );
return false;
}
}
@@ -2715,79 +2715,79 @@ bool TReadInitAndAuthActor::CheckTopicACL(
}
-void TReadInitAndAuthActor::CheckClientACL(const TActorContext& ctx) {
- // ToDo[migration] - Through converter/metacache - ?
- SendCacheNavigateRequest(ctx, AppData(ctx)->PQConfig.GetRoot() + "/" + ClientPath);
-}
+void TReadInitAndAuthActor::CheckClientACL(const TActorContext& ctx) {
+ // ToDo[migration] - Through converter/metacache - ?
+ SendCacheNavigateRequest(ctx, AppData(ctx)->PQConfig.GetRoot() + "/" + ClientPath);
+}
-void TReadInitAndAuthActor::HandleClientSchemeCacheResponse(
- TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx
-) {
- TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get();
- const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get();
+void TReadInitAndAuthActor::HandleClientSchemeCacheResponse(
+ TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx
+) {
+ TEvTxProxySchemeCache::TEvNavigateKeySetResult* msg = ev->Get();
+ const NSchemeCache::TSchemeCacheNavigate* navigate = msg->Request.Get();
- Y_VERIFY(navigate->ResultSet.size() == 1);
- auto& entry = navigate->ResultSet.front();
- auto path = "/" + JoinPath(entry.Path); // ToDo [migration] - through converter ?
- if (navigate->ErrorCount > 0) {
- const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status;
- CloseSession(TStringBuilder() << "Failed to read ACL for '" << path << "' Scheme cache error : " << status, PersQueue::ErrorCode::ERROR, ctx);
- return;
+ Y_VERIFY(navigate->ResultSet.size() == 1);
+ auto& entry = navigate->ResultSet.front();
+ auto path = "/" + JoinPath(entry.Path); // ToDo [migration] - through converter ?
+ if (navigate->ErrorCount > 0) {
+ const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status;
+ CloseSession(TStringBuilder() << "Failed to read ACL for '" << path << "' Scheme cache error : " << status, PersQueue::ErrorCode::ERROR, ctx);
+ return;
}
- NACLib::EAccessRights rights = (NACLib::EAccessRights)(NACLib::EAccessRights::ReadAttributes + NACLib::EAccessRights::WriteAttributes);
- if (
- !CheckACLPermissionsForNavigate(entry.SecurityObject, path, rights, "No ReadAsConsumer permissions", ctx)
- ) {
+ NACLib::EAccessRights rights = (NACLib::EAccessRights)(NACLib::EAccessRights::ReadAttributes + NACLib::EAccessRights::WriteAttributes);
+ if (
+ !CheckACLPermissionsForNavigate(entry.SecurityObject, path, rights, "No ReadAsConsumer permissions", ctx)
+ ) {
return;
- }
- FinishInitialization(ctx);
-}
-
-
-bool TReadInitAndAuthActor::CheckACLPermissionsForNavigate(
- const TIntrusivePtr<TSecurityObject>& secObject, const TString& path,
- NACLib::EAccessRights rights, const TString& errorTextWhenAccessDenied, const TActorContext& ctx
-) {
- // TODO: SCHEME_ERROR если нет топика/консумера
- // TODO: если AccessDenied на корень, то надо ACCESS_DENIED, а не SCHEME_ERROR
-
- if (DoCheckACL && !secObject->CheckAccess(rights, *Token)) {
- CloseSession(
- TStringBuilder() << errorTextWhenAccessDenied << " for '" << path
- << "' for subject '" << Token->GetUserSID() << "'",
- PersQueue::ErrorCode::ACCESS_DENIED, ctx
- );
- return false;
- }
- return true;
-}
-
-
-void TReadInitAndAuthActor::FinishInitialization(const TActorContext& ctx) {
- TTopicTabletsPairs res;
- for (auto& t : Topics) {
- res.emplace_back(decltype(res)::value_type({
- t.second.TopicNameConverter, t.second.TabletID, t.second.CloudId, t.second.DbId, t.second.FolderId
- }));
- }
- ctx.Send(ParentId, new TEvPQProxy::TEvAuthResultOk(std::move(res)));
- Die(ctx);
+ }
+ FinishInitialization(ctx);
+}
+
+
+bool TReadInitAndAuthActor::CheckACLPermissionsForNavigate(
+ const TIntrusivePtr<TSecurityObject>& secObject, const TString& path,
+ NACLib::EAccessRights rights, const TString& errorTextWhenAccessDenied, const TActorContext& ctx
+) {
+ // TODO: SCHEME_ERROR если нет топика/консумера
+ // TODO: если AccessDenied на корень, то надо ACCESS_DENIED, а не SCHEME_ERROR
+
+ if (DoCheckACL && !secObject->CheckAccess(rights, *Token)) {
+ CloseSession(
+ TStringBuilder() << errorTextWhenAccessDenied << " for '" << path
+ << "' for subject '" << Token->GetUserSID() << "'",
+ PersQueue::ErrorCode::ACCESS_DENIED, ctx
+ );
+ return false;
+ }
+ return true;
+}
+
+
+void TReadInitAndAuthActor::FinishInitialization(const TActorContext& ctx) {
+ TTopicTabletsPairs res;
+ for (auto& t : Topics) {
+ res.emplace_back(decltype(res)::value_type({
+ t.second.TopicNameConverter, t.second.TabletID, t.second.CloudId, t.second.DbId, t.second.FolderId
+ }));
+ }
+ ctx.Send(ParentId, new TEvPQProxy::TEvAuthResultOk(std::move(res)));
+ Die(ctx);
}
// READINFOACTOR
-TReadInfoActor::TReadInfoActor(
- TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler,
- const TActorId& schemeCache, const TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters
-)
+TReadInfoActor::TReadInfoActor(
+ TEvPQReadInfoRequest* request, const NPersQueue::TTopicsListController& topicsHandler,
+ const TActorId& schemeCache, const TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters
+)
: TBase(request)
, SchemeCache(schemeCache)
, NewSchemeCache(newSchemeCache)
, AuthInitActor()
, Counters(counters)
- , TopicsHandler(topicsHandler)
+ , TopicsHandler(topicsHandler)
{
Y_ASSERT(request);
}
@@ -2820,18 +2820,18 @@ void TReadInfoActor::Bootstrap(const TActorContext& ctx) {
THashSet<TString> topicsToResolve;
for (auto& t : request->topics()) {
- if (t.path().empty()) {
+ if (t.path().empty()) {
AnswerError("empty topic in init request", PersQueue::ErrorCode::BAD_REQUEST, ctx);
return;
}
- topicsToResolve.insert(t.path());
+ topicsToResolve.insert(t.path());
}
- AuthInitActor = ctx.Register(new TReadInitAndAuthActor(
- ctx, ctx.SelfID, ClientId, 0, TString("read_info:") + Request().GetPeerName(),
- SchemeCache, NewSchemeCache, Counters, token,
- TopicsHandler.GetReadTopicsList(topicsToResolve, readOnlyLocal, Request().GetDatabaseName().GetOrElse(TString()))
- ));
+ AuthInitActor = ctx.Register(new TReadInitAndAuthActor(
+ ctx, ctx.SelfID, ClientId, 0, TString("read_info:") + Request().GetPeerName(),
+ SchemeCache, NewSchemeCache, Counters, token,
+ TopicsHandler.GetReadTopicsList(topicsToResolve, readOnlyLocal, Request().GetDatabaseName().GetOrElse(TString()))
+ ));
}
@@ -2846,7 +2846,7 @@ void TReadInfoActor::Die(const TActorContext& ctx) {
void TReadInfoActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) {
LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "GetReadInfo auth ok fo read info, got " << ev->Get()->TopicAndTablets.size() << " topics");
- TopicAndTablets = std::move(ev->Get()->TopicAndTablets);
+ TopicAndTablets = std::move(ev->Get()->TopicAndTablets);
if (TopicAndTablets.empty()) {
AnswerError("empty list of topics", PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx);
return;
@@ -2855,7 +2855,7 @@ void TReadInfoActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorC
NKikimrClient::TPersQueueRequest proto;
proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->SetClientId(ClientId);
for (auto& t : TopicAndTablets) {
- proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(t.TopicNameConverter->GetClientsideName());
+ proto.MutableMetaRequest()->MutableCmdGetReadSessionsInfo()->AddTopic(t.TopicNameConverter->GetClientsideName());
}
ctx.Register(NMsgBusProxy::CreateActorServerPersQueue(
diff --git a/ydb/services/persqueue_v1/grpc_pq_schema.cpp b/ydb/services/persqueue_v1/grpc_pq_schema.cpp
index c844a712d3..7756adc797 100644
--- a/ydb/services/persqueue_v1/grpc_pq_schema.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_schema.cpp
@@ -40,7 +40,7 @@ TPQSchemaService::TPQSchemaService(const TActorId& schemeCache,
void TPQSchemaService::Bootstrap(const TActorContext& ctx) {
- if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration]: switch to haveClusters
+ if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration]: switch to haveClusters
ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(),
new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe);
}
@@ -136,7 +136,7 @@ void TDescribeTopicActor::HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEv
auto settings = result.mutable_settings();
Ydb::Scheme::Entry *selfEntry = result.mutable_self();
const auto& selfInfo = response.Self->Info;
- selfEntry->set_name(path + "/" + selfInfo.GetName());
+ selfEntry->set_name(path + "/" + selfInfo.GetName());
selfEntry->set_type(static_cast<Ydb::Scheme::Entry::Type>(selfInfo.GetPathType()));
ConvertDirectoryEntry(selfInfo, selfEntry, true);
if (response.PQGroupInfo) {
diff --git a/ydb/services/persqueue_v1/grpc_pq_write.cpp b/ydb/services/persqueue_v1/grpc_pq_write.cpp
index dc9f9eaf27..5f17a2f3ab 100644
--- a/ydb/services/persqueue_v1/grpc_pq_write.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_write.cpp
@@ -37,15 +37,15 @@ TPQWriteService::TPQWriteService(const TActorId& schemeCache, const TActorId& ne
void TPQWriteService::Bootstrap(const TActorContext& ctx) {
- HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration]: switch to proper option
- if (HaveClusters) {
+ HaveClusters = !AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(); // ToDo[migration]: switch to proper option
+ if (HaveClusters) {
ctx.Send(NPQ::NClusterTracker::MakeClusterTrackerID(),
new NPQ::NClusterTracker::TEvClusterTracker::TEvSubscribe);
}
ctx.Send(NNetClassifier::MakeNetClassifierID(), new NNetClassifier::TEvNetClassifier::TEvSubscribe);
- ConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>(
- AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot()
- );
+ ConverterFactory = std::make_shared<NPersQueue::TTopicNamesConverterFactory>(
+ AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen(), AppData(ctx)->PQConfig.GetRoot()
+ );
Become(&TThis::StateFunc);
}
@@ -178,7 +178,7 @@ void TPQWriteService::Handle(NKikimr::NGRpcService::TEvStreamPQWriteRequest::TPt
TString localCluster = AvailableLocalCluster(ctx);
- if (HaveClusters && localCluster.empty()) {
+ if (HaveClusters && localCluster.empty()) {
ev->Get()->GetStreamCtx()->Attach(ctx.SelfID);
if (LocalCluster) {
LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "new grpc connection failed - cluster disabled");
@@ -189,19 +189,19 @@ void TPQWriteService::Handle(NKikimr::NGRpcService::TEvStreamPQWriteRequest::TPt
}
return;
} else {
- TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>(
- ConverterFactory, HaveClusters, TVector<TString>{}, localCluster
- );
+ TopicsHandler = std::make_unique<NPersQueue::TTopicsListController>(
+ ConverterFactory, HaveClusters, TVector<TString>{}, localCluster
+ );
const ui64 cookie = NextCookie();
LOG_DEBUG_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "new session created cookie " << cookie);
auto ip = ev->Get()->GetStreamCtx()->GetPeerName();
- TActorId worker = ctx.Register(new TWriteSessionActor(
- ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters,
- DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown",
- *TopicsHandler
- ));
+ TActorId worker = ctx.Register(new TWriteSessionActor(
+ ev->Release().Release(), cookie, SchemeCache, NewSchemeCache, Counters,
+ DatacenterClassifier ? DatacenterClassifier->ClassifyAddress(NAddressClassifier::ExtractAddress(ip)) : "unknown",
+ *TopicsHandler
+ ));
Sessions[cookie] = worker;
}
@@ -212,8 +212,8 @@ bool TPQWriteService::TooMuchSessions() {
}
-TString TPQWriteService::AvailableLocalCluster(const TActorContext&) const {
- return HaveClusters && Enabled ? *LocalCluster : "";
+TString TPQWriteService::AvailableLocalCluster(const TActorContext&) const {
+ return HaveClusters && Enabled ? *LocalCluster : "";
}
diff --git a/ydb/services/persqueue_v1/grpc_pq_write.h b/ydb/services/persqueue_v1/grpc_pq_write.h
index bb0af21dee..73148388b2 100644
--- a/ydb/services/persqueue_v1/grpc_pq_write.h
+++ b/ydb/services/persqueue_v1/grpc_pq_write.h
@@ -81,9 +81,9 @@ private:
TString DeleteSourceIdQuery;
NAddressClassifier::TLabeledAddressClassifier::TConstPtr DatacenterClassifier; // Detects client's datacenter by IP. May be null
- bool HaveClusters;
- NPersQueue::TConverterFactoryPtr ConverterFactory;
- std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler;
+ bool HaveClusters;
+ NPersQueue::TConverterFactoryPtr ConverterFactory;
+ std::unique_ptr<NPersQueue::TTopicsListController> TopicsHandler;
};
diff --git a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
index 5fba4e8686..3038cc82a6 100644
--- a/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
+++ b/ydb/services/persqueue_v1/grpc_pq_write_actor.cpp
@@ -22,7 +22,7 @@ using namespace NKikimrClient;
namespace NKikimr {
-using namespace NSchemeCache;
+using namespace NSchemeCache;
Ydb::PersQueue::V1::Codec CodecByName(const TString& codec) {
static const THashMap<TString, Ydb::PersQueue::V1::Codec> codecsByName = {
@@ -108,19 +108,19 @@ static const TString UPDATE_SOURCEID_QUERY2 = "` (Hash, Topic, SourceId, CreateT
//TODO: add here tracking of bytes in/out
-TWriteSessionActor::TWriteSessionActor(
- NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie,
- const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
- TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
- const NPersQueue::TTopicsListController& topicsController
-)
+TWriteSessionActor::TWriteSessionActor(
+ NKikimr::NGRpcService::TEvStreamPQWriteRequest* request, const ui64 cookie,
+ const NActors::TActorId& schemeCache, const NActors::TActorId& newSchemeCache,
+ TIntrusivePtr<NMonitoring::TDynamicCounters> counters, const TMaybe<TString> clientDC,
+ const NPersQueue::TTopicsListController& topicsController
+)
: Request(request)
, State(ES_CREATED)
, SchemeCache(schemeCache)
, NewSchemeCache(newSchemeCache)
, PeerName("")
, Cookie(cookie)
- , TopicsController(topicsController)
+ , TopicsController(topicsController)
, Partition(0)
, PreferedPartition(Max<ui32>())
, NumReserveBytesRequests(0)
@@ -174,23 +174,23 @@ void TWriteSessionActor::HandleDone(const TActorContext& ctx) {
Die(ctx);
}
-TString WriteRequestToLog(const Ydb::PersQueue::V1::StreamingWriteClientMessage& proto) {
- switch (proto.client_message_case()) {
- case StreamingWriteClientMessage::kInitRequest:
- return proto.ShortDebugString();
- break;
- case StreamingWriteClientMessage::kWriteRequest:
- return " write_request[data omitted]";
- break;
- case StreamingWriteClientMessage::kUpdateTokenRequest:
- return " update_token_request [content omitted]";
- default:
- return TString();
- }
-}
-
+TString WriteRequestToLog(const Ydb::PersQueue::V1::StreamingWriteClientMessage& proto) {
+ switch (proto.client_message_case()) {
+ case StreamingWriteClientMessage::kInitRequest:
+ return proto.ShortDebugString();
+ break;
+ case StreamingWriteClientMessage::kWriteRequest:
+ return " write_request[data omitted]";
+ break;
+ case StreamingWriteClientMessage::kUpdateTokenRequest:
+ return " update_token_request [content omitted]";
+ default:
+ return TString();
+ }
+}
+
void TWriteSessionActor::Handle(IContext::TEvReadFinished::TPtr& ev, const TActorContext& ctx) {
- LOG_DEBUG_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read done: success: " << ev->Get()->Success << " data: " << WriteRequestToLog(ev->Get()->Record));
+ LOG_DEBUG_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read done: success: " << ev->Get()->Success << " data: " << WriteRequestToLog(ev->Get()->Record));
if (!ev->Get()->Success) {
LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " grpc read failed");
ctx.Send(ctx.SelfID, new TEvPQProxy::TEvDone());
@@ -260,7 +260,7 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvDone::TPtr&, const TActorContext&
}
void TWriteSessionActor::CheckACL(const TActorContext& ctx) {
- //Y_VERIFY(ACLCheckInProgress);
+ //Y_VERIFY(ACLCheckInProgress);
NACLib::EAccessRights rights = NACLib::EAccessRights::UpdateRow;
@@ -283,7 +283,7 @@ void TWriteSessionActor::CheckACL(const TActorContext& ctx) {
}
} else {
TString errorReason = Sprintf("access to topic '%s' denied for '%s' due to 'no WriteTopic rights', Marker# PQ1125",
- TopicConverter->GetClientsideName().c_str(),
+ TopicConverter->GetClientsideName().c_str(),
Token->GetUserSID().c_str());
CloseSession(errorReason, PersQueue::ErrorCode::ACCESS_DENIED, ctx);
}
@@ -304,15 +304,15 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor
return;
}
- TopicConverter = TopicsController.GetWriteTopicConverter(init.topic(), Request->GetDatabaseName().GetOrElse("/Root"));
- if (!TopicConverter->IsValid()) {
- CloseSession(
- TStringBuilder() << "topic " << init.topic() << " could not be recognized: " << TopicConverter->GetReason(),
- PersQueue::ErrorCode::BAD_REQUEST, ctx
- );
- return;
+ TopicConverter = TopicsController.GetWriteTopicConverter(init.topic(), Request->GetDatabaseName().GetOrElse("/Root"));
+ if (!TopicConverter->IsValid()) {
+ CloseSession(
+ TStringBuilder() << "topic " << init.topic() << " could not be recognized: " << TopicConverter->GetReason(),
+ PersQueue::ErrorCode::BAD_REQUEST, ctx
+ );
+ return;
}
-
+
PeerName = event->PeerName;
SourceId = init.message_group_id();
@@ -325,7 +325,7 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor
}
EscapedSourceId = HexEncode(encodedSourceId);
- TString s = TopicConverter->GetClientsideName() + encodedSourceId;
+ TString s = TopicConverter->GetClientsideName() + encodedSourceId;
Hash = MurmurHash<ui32>(s.c_str(), s.size(), MURMUR_ARRAY_SEED);
LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session request cookie: " << Cookie << " " << init << " from " << PeerName);
@@ -345,10 +345,10 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor
PreferedPartition = init.partition_group_id() > 0 ? init.partition_group_id() - 1 : Max<ui32>();
- InitMeta = GetInitialDataChunk(init, TopicConverter->GetFullLegacyName(), PeerName); // ToDo[migration] - check?
+ InitMeta = GetInitialDataChunk(init, TopicConverter->GetFullLegacyName(), PeerName); // ToDo[migration] - check?
auto subGroup = GetServiceCounters(Counters, "pqproxy|SLI");
- Aggr = {{{{"Account", TopicConverter->GetAccount()}}, {"total"}}};
+ Aggr = {{{{"Account", TopicConverter->GetAccount()}}, {"total"}}};
SLITotal = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsTotal"}, true, "sensor", false);
SLIErrors = NKikimr::NPQ::TMultiCounter(subGroup, Aggr, {}, {"RequestsError"}, true, "sensor", false);
@@ -364,7 +364,7 @@ void TWriteSessionActor::SetupCounters()
{
//now topic is checked, can create group for real topic, not garbage
auto subGroup = GetServiceCounters(Counters, "pqproxy|writeSession");
- TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(LocalDC, TopicConverter->GetClientsideName());
+ TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabels(LocalDC, TopicConverter->GetClientsideName());
BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflight"}, false);
BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"BytesInflightTotal"}, false);
@@ -381,7 +381,7 @@ void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& db
{
//now topic is checked, can create group for real topic, not garbage
auto subGroup = NKikimr::NPQ::GetCountersForStream(Counters, "writeSession");
- TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(TopicConverter->GetClientsideName(), cloudId, dbId, folderId);
+ TVector<NPQ::TLabelsInfo> aggr = NKikimr::NPQ::GetLabelsForStream(TopicConverter->GetClientsideName(), cloudId, dbId, folderId);
BytesInflight = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding"}, false);
BytesInflightTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"stream.internal_write.bytes_proceeding_total"}, false);
@@ -396,35 +396,35 @@ void TWriteSessionActor::SetupCounters(const TString& cloudId, const TString& db
void TWriteSessionActor::InitCheckSchema(const TActorContext& ctx, bool needWaitSchema) {
LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "init check schema");
- if (!needWaitSchema) {
- ACLCheckInProgress = true;
+ if (!needWaitSchema) {
+ ACLCheckInProgress = true;
}
- ctx.Send(SchemeCache, new TEvDescribeTopicsRequest({TopicConverter->GetPrimaryPath()}));
+ ctx.Send(SchemeCache, new TEvDescribeTopicsRequest({TopicConverter->GetPrimaryPath()}));
if (needWaitSchema) {
State = ES_WAIT_SCHEME_2;
}
}
-void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) {
- auto* res = ev->Get()->Result.Get();
- Y_VERIFY(res->ResultSet.size() == 1);
+void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActorContext& ctx) {
+ auto* res = ev->Get()->Result.Get();
+ Y_VERIFY(res->ResultSet.size() == 1);
- auto& entry = res->ResultSet[0];
+ auto& entry = res->ResultSet[0];
TString errorReason;
- auto processResult = ProcessMetaCacheTopicResponse(entry);
- if (processResult.IsFatal) {
- CloseSession(processResult.Reason, processResult.ErrorCode, ctx);
+ auto processResult = ProcessMetaCacheTopicResponse(entry);
+ if (processResult.IsFatal) {
+ CloseSession(processResult.Reason, processResult.ErrorCode, ctx);
return;
}
- auto& description = entry.PQGroupInfo->Description;
- Y_VERIFY(description.PartitionsSize() > 0);
- Y_VERIFY(description.HasPQTabletConfig());
- InitialPQTabletConfig = description.GetPQTabletConfig();
+ auto& description = entry.PQGroupInfo->Description;
+ Y_VERIFY(description.PartitionsSize() > 0);
+ Y_VERIFY(description.HasPQTabletConfig());
+ InitialPQTabletConfig = description.GetPQTabletConfig();
- BalancerTabletId = description.GetBalancerTabletID();
+ BalancerTabletId = description.GetBalancerTabletID();
- for (ui32 i = 0; i < description.PartitionsSize(); ++i) {
- const auto& pi = description.GetPartitions(i);
+ for (ui32 i = 0; i < description.PartitionsSize(); ++i) {
+ const auto& pi = description.GetPartitions(i);
PartitionToTablet[pi.GetPartitionId()] = pi.GetTabletId();
}
@@ -436,10 +436,10 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo
SetupCounters();
}
- Y_VERIFY (entry.SecurityObject);
- ACL.Reset(new TAclWrapper(entry.SecurityObject));
- LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check");
-
+ Y_VERIFY (entry.SecurityObject);
+ ACL.Reset(new TAclWrapper(entry.SecurityObject));
+ LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session v1 cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check");
+
if (Request->GetInternalToken().empty()) { // session without auth
if (AppData(ctx)->PQConfig.GetRequireCredentialsInNewProtocol()) {
Request->ReplyUnauthenticated("Unauthenticated access is forbidden, please provide credentials");
@@ -454,7 +454,7 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo
Auth = *Request->GetYdbToken();
Token = new NACLib::TUserToken(Request->GetInternalToken());
- CheckACL(ctx);
+ CheckACL(ctx);
}
}
@@ -464,11 +464,11 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::
Y_VERIFY(navigate->ResultSet.size() == 1);
if (navigate->ErrorCount > 0) {
const NSchemeCache::TSchemeCacheNavigate::EStatus status = navigate->ResultSet.front().Status;
- return CloseSession(
- TStringBuilder() << "Failed to read ACL for '" << TopicConverter->GetClientsideName()
- << "' Scheme cache error : " << status,
- PersQueue::ErrorCode::ERROR, ctx
- );
+ return CloseSession(
+ TStringBuilder() << "Failed to read ACL for '" << TopicConverter->GetClientsideName()
+ << "' Scheme cache error : " << status,
+ PersQueue::ErrorCode::ERROR, ctx
+ );
}
const auto& pqDescription = navigate->ResultSet.front().PQGroupInfo->Description;
@@ -478,7 +478,7 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::
InitialPQTabletConfig = pqDescription.GetPQTabletConfig();
if (!pqDescription.HasBalancerTabletID()) {
- TString errorReason = Sprintf("topic '%s' has no balancer, Marker# PQ93", TopicConverter->GetClientsideName().c_str());
+ TString errorReason = Sprintf("topic '%s' has no balancer, Marker# PQ93", TopicConverter->GetClientsideName().c_str());
CloseSession(errorReason, PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx);
return;
}
@@ -515,7 +515,7 @@ void TWriteSessionActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::
void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) {
- if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration] - separate flag for having config tables
+ if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { // ToDo[migration] - separate flag for having config tables
auto partitionId = PreferedPartition < Max<ui32>() ? PreferedPartition
: NKikimr::NDataStreams::V1::ShardFromDecimal(NKikimr::NDataStreams::V1::HexBytesToDecimal(MD5::Calc(SourceId)), PartitionToTablet.size());
ProceedPartition(partitionId, ctx);
@@ -528,7 +528,7 @@ void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) {
ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
ev->Record.MutableRequest()->SetKeepSession(false);
ev->Record.MutableRequest()->SetQuery(SelectSourceIdQuery);
- ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig));
+ ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig));
// fill tx settings: set commit tx flag & begin new serializable tx.
ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
ev->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write();
@@ -536,7 +536,7 @@ void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) {
ev->Record.MutableRequest()->MutableQueryCachePolicy()->set_keep_in_cache(true);
NClient::TParameters parameters;
parameters["$Hash"] = Hash;
- parameters["$Topic"] = TopicConverter->GetClientsideName();
+ parameters["$Topic"] = TopicConverter->GetClientsideName();
parameters["$SourceId"] = EscapedSourceId;
ev->Record.MutableRequest()->MutableParameters()->Swap(&parameters);
ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
@@ -546,7 +546,7 @@ void TWriteSessionActor::DiscoverPartition(const NActors::TActorContext& ctx) {
void TWriteSessionActor::UpdatePartition(const TActorContext& ctx) {
Y_VERIFY(State == ES_WAIT_TABLE_REQUEST_1 || State == ES_WAIT_NEXT_PARTITION);
- auto ev = MakeUpdateSourceIdMetadataRequest(ctx);
+ auto ev = MakeUpdateSourceIdMetadataRequest(ctx);
ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
State = ES_WAIT_TABLE_REQUEST_2;
}
@@ -644,16 +644,16 @@ void TWriteSessionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const
}
}
-THolder<NKqp::TEvKqp::TEvQueryRequest> TWriteSessionActor::MakeUpdateSourceIdMetadataRequest(
- const NActors::TActorContext& ctx
-) {
-
+THolder<NKqp::TEvKqp::TEvQueryRequest> TWriteSessionActor::MakeUpdateSourceIdMetadataRequest(
+ const NActors::TActorContext& ctx
+) {
+
auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML);
ev->Record.MutableRequest()->SetQuery(UpdateSourceIdQuery);
- ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig));
+ ev->Record.MutableRequest()->SetDatabase(NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig));
ev->Record.MutableRequest()->SetKeepSession(false);
// fill tx settings: set commit tx flag & begin new serializable tx.
ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true);
@@ -663,7 +663,7 @@ THolder<NKqp::TEvKqp::TEvQueryRequest> TWriteSessionActor::MakeUpdateSourceIdMet
NClient::TParameters parameters;
parameters["$Hash"] = Hash;
- parameters["$Topic"] = TopicConverter->GetClientsideName();
+ parameters["$Topic"] = TopicConverter->GetClientsideName();
parameters["$SourceId"] = EscapedSourceId;
parameters["$CreateTime"] = SourceIdCreateTime;
parameters["$AccessTime"] = TInstant::Now().MilliSeconds();
@@ -691,10 +691,10 @@ void TWriteSessionActor::ProceedPartition(const ui32 partition, const TActorCont
ui64 tabletId = it != PartitionToTablet.end() ? it->second : 0;
if (!tabletId) {
- CloseSession(
- Sprintf("no partition %u in topic '%s', Marker# PQ4", Partition, TopicConverter->GetClientsideName().c_str()),
- PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx
- );
+ CloseSession(
+ Sprintf("no partition %u in topic '%s', Marker# PQ4", Partition, TopicConverter->GetClientsideName().c_str()),
+ PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx
+ );
return;
}
@@ -770,8 +770,8 @@ void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev
init->set_session_id(EscapeC(OwnerCookie));
init->set_last_sequence_number(maxSeqNo);
init->set_partition_id(Partition);
- init->set_topic(TopicConverter->GetModernName());
- init->set_cluster(TopicConverter->GetCluster());
+ init->set_topic(TopicConverter->GetModernName());
+ init->set_cluster(TopicConverter->GetCluster());
init->set_block_format_version(0);
if (InitialPQTabletConfig.HasCodecs()) {
for (const auto& codecName : InitialPQTabletConfig.GetCodecs().GetCodecs()) {
@@ -1146,7 +1146,7 @@ void TWriteSessionActor::HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const
void TWriteSessionActor::LogSession(const TActorContext& ctx) {
LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "write session: cookie=" << Cookie << " sessionId=" << OwnerCookie << " userAgent=\"" << UserAgent << "\" ip=" << PeerName << " proto=v1 "
- << " topic=" << TopicConverter->GetModernName() << " durationSec=" << (ctx.Now() - StartTime).Seconds());
+ << " topic=" << TopicConverter->GetModernName() << " durationSec=" << (ctx.Now() - StartTime).Seconds());
LogSessionDeadline = ctx.Now() + TDuration::Hours(1) + TDuration::Seconds(rand() % 60);
}
@@ -1158,9 +1158,9 @@ void TWriteSessionActor::HandleWakeup(const TActorContext& ctx) {
RequestNotChecked = false;
InitCheckSchema(ctx);
}
- // ToDo[migration] - separate flag for having config tables
+ // ToDo[migration] - separate flag for having config tables
if (!AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen() && !SourceIdUpdateInfly && ctx.Now() - LastSourceIdUpdate > SOURCEID_UPDATE_PERIOD) {
- auto ev = MakeUpdateSourceIdMetadataRequest(ctx);
+ auto ev = MakeUpdateSourceIdMetadataRequest(ctx);
SourceIdUpdateInfly = true;
ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
}
diff --git a/ydb/services/persqueue_v1/persqueue_common_ut.cpp b/ydb/services/persqueue_v1/persqueue_common_ut.cpp
index a4d32e199d..9d706d0656 100644
--- a/ydb/services/persqueue_v1/persqueue_common_ut.cpp
+++ b/ydb/services/persqueue_v1/persqueue_common_ut.cpp
@@ -36,32 +36,32 @@ using namespace Ydb::PersQueue::V1;
using namespace NThreading;
using namespace NNetClassifier;
-
-#define MAKE_WRITE_STREAM(TOKEN) \
- grpc::ClientContext context; \
- context.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, TOKEN); \
- auto stream = server.ServiceStub->StreamingWrite(&context); \
-
-
+
+#define MAKE_WRITE_STREAM(TOKEN) \
+ grpc::ClientContext context; \
+ context.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, TOKEN); \
+ auto stream = server.ServiceStub->StreamingWrite(&context); \
+
+
Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
// Auth* tests are for both authentication and authorization
- Y_UNIT_TEST(Auth_CreateGrpcStreamWithInvalidTokenInInitialMetadata_SessionClosedWithUnauthenticatedError) {
- TPersQueueV1TestServer server;
- SET_LOCALS;
-
- runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
- runtime->GetAppData().EnforceUserTokenRequirement = true;
+ Y_UNIT_TEST(Auth_CreateGrpcStreamWithInvalidTokenInInitialMetadata_SessionClosedWithUnauthenticatedError) {
+ TPersQueueV1TestServer server;
+ SET_LOCALS;
+
+ runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
+ runtime->GetAppData().EnforceUserTokenRequirement = true;
TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"};
for (const auto &invalidToken : invalidTokens) {
Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl;
- MAKE_WRITE_STREAM(invalidToken);
+ MAKE_WRITE_STREAM(invalidToken);
// TODO: Message should be written to gRPC in order to get error. Fix gRPC data plane API code if our expectations are different.
// Note that I check that initial metadata is sent during gRPC stream constructor.
StreamingWriteClientMessage clientMessage;
StreamingWriteServerMessage serverMessage;
- clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
+ clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
clientMessage.mutable_init_request()->set_message_group_id("test-group-id");
AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage);
@@ -76,22 +76,22 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
return "test_user_" + ToString(i) + "@" + BUILTIN_ACL_DOMAIN;
}
- Y_UNIT_TEST(Auth_MultipleUpdateTokenRequestIterationsWithValidToken_GotUpdateTokenResponseForEachRequest) {
- TPersQueueV1TestServer server;
-
+ Y_UNIT_TEST(Auth_MultipleUpdateTokenRequestIterationsWithValidToken_GotUpdateTokenResponseForEachRequest) {
+ TPersQueueV1TestServer server;
+
const int iterations = 10;
NACLib::TDiffACL acl;
for (int i = 0; i != iterations; ++i) {
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i));
}
- server.ModifyTopicACL(server.GetTopic(), acl);
+ server.ModifyTopicACL(server.GetTopic(), acl);
- MAKE_WRITE_STREAM(GenerateValidToken(0));
+ MAKE_WRITE_STREAM(GenerateValidToken(0));
StreamingWriteClientMessage clientMessage;
StreamingWriteServerMessage serverMessage;
- clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
+ clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
clientMessage.mutable_init_request()->set_message_group_id("test-group-id");
AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage);
AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream);
@@ -109,22 +109,22 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
}
}
- Y_UNIT_TEST(
- Auth_WriteSessionWithValidTokenAndACEAndThenRemoveACEAndSendWriteRequest_SessionClosedWithUnauthorizedErrorAfterSuccessfullWriteResponse
- ) {
- TPersQueueV1TestServer server;
-
+ Y_UNIT_TEST(
+ Auth_WriteSessionWithValidTokenAndACEAndThenRemoveACEAndSendWriteRequest_SessionClosedWithUnauthorizedErrorAfterSuccessfullWriteResponse
+ ) {
+ TPersQueueV1TestServer server;
+
//setup.GetPQConfig().SetACLRetryTimeoutSec(0);
NACLib::TDiffACL acl;
const auto token = GenerateValidToken();
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, token);
- server.ModifyTopicACL(server.GetTopic(), acl);
+ server.ModifyTopicACL(server.GetTopic(), acl);
- MAKE_WRITE_STREAM(token);
+ MAKE_WRITE_STREAM(token);
StreamingWriteClientMessage clientMessage;
StreamingWriteServerMessage serverMessage;
- clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
+ clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
clientMessage.mutable_init_request()->set_message_group_id("test-group-id");
AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage);
AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream);
@@ -132,7 +132,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
serverMessage);
acl.ClearAccess();
- server.ModifyTopicACL(server.GetTopic(), acl);
+ server.ModifyTopicACL(server.GetTopic(), acl);
clientMessage = StreamingWriteClientMessage();
auto *writeRequest = clientMessage.mutable_write_request();
@@ -158,23 +158,23 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
}
// TODO: Replace this test with a unit-test of TWriteSessionActor
- Y_UNIT_TEST(
- Auth_MultipleInflightWriteUpdateTokenRequestWithDifferentValidToken_SessionClosedWithOverloadedError
- ) {
- TPersQueueV1TestServer server;
- SET_LOCALS;
+ Y_UNIT_TEST(
+ Auth_MultipleInflightWriteUpdateTokenRequestWithDifferentValidToken_SessionClosedWithOverloadedError
+ ) {
+ TPersQueueV1TestServer server;
+ SET_LOCALS;
const int iterations = 3;
NACLib::TDiffACL acl;
for (int i = 0; i != iterations; ++i) {
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, GenerateValidToken(i));
}
- server.ModifyTopicACL(server.GetTopic(), acl);
- MAKE_WRITE_STREAM(GenerateValidToken(0));
+ server.ModifyTopicACL(server.GetTopic(), acl);
+ MAKE_WRITE_STREAM(GenerateValidToken(0));
StreamingWriteClientMessage clientMessage;
StreamingWriteServerMessage serverMessage;
- clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
+ clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
clientMessage.mutable_init_request()->set_message_group_id("test-group-id");
AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage);
AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream);
@@ -183,9 +183,9 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
// TWriteSessionActor uses GRpcRequestProxy for authentication. This will make next update token procedure stuck indefinetely
auto noopActorID = TActorId();
- for (size_t i = 0; i != runtime->GetNodeCount(); ++i) {
+ for (size_t i = 0; i != runtime->GetNodeCount(); ++i) {
// Destroy GRpcRequestProxy
- runtime->RegisterService(NGRpcService::CreateGRpcRequestProxyId(), noopActorID);
+ runtime->RegisterService(NGRpcService::CreateGRpcRequestProxyId(), noopActorID);
}
clientMessage.mutable_update_token_request()->set_token(GenerateValidToken(1));
@@ -203,24 +203,24 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
serverMessage);
}
- Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithInvalidToken_SessionClosedWithUnauthenticatedError) {
- TPersQueueV1TestServer server;
- SET_LOCALS;
- runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
+ Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithInvalidToken_SessionClosedWithUnauthenticatedError) {
+ TPersQueueV1TestServer server;
+ SET_LOCALS;
+ runtime->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
const TString validToken = "test_user@" BUILTIN_ACL_DOMAIN;
// TODO: Why test fails with 'BUILTIN_ACL_DOMAIN' as domain in invalid token?
TVector<TString> invalidTokens = {TString(), "test_user", "test_user@invalid_domain"};
NACLib::TDiffACL acl;
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken);
- server.ModifyTopicACL(server.GetTopic(), acl);
+ server.ModifyTopicACL(server.GetTopic(), acl);
for (const auto &invalidToken : invalidTokens) {
Cerr << "Invalid token under test is '" << invalidToken << "'" << Endl;
- MAKE_WRITE_STREAM(validToken);
+ MAKE_WRITE_STREAM(validToken);
StreamingWriteClientMessage clientMessage;
StreamingWriteServerMessage serverMessage;
- clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
+ clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
clientMessage.mutable_init_request()->set_message_group_id("test-group-id");
AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage);
AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream);
@@ -238,9 +238,9 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
}
}
- Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithValidTokenButWithoutACL_SessionClosedWithUnauthorizedError) {
- TPersQueueV1TestServer server;
-
+ Y_UNIT_TEST(Auth_WriteUpdateTokenRequestWithValidTokenButWithoutACL_SessionClosedWithUnauthorizedError) {
+ TPersQueueV1TestServer server;
+
const TString validToken = "test_user@"
BUILTIN_ACL_DOMAIN;
const TString invalidToken = "test_user_2@"
@@ -248,13 +248,13 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
NACLib::TDiffACL acl;
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, validToken);
- server.ModifyTopicACL(server.GetTopic(), acl);
+ server.ModifyTopicACL(server.GetTopic(), acl);
- MAKE_WRITE_STREAM(validToken);
+ MAKE_WRITE_STREAM(validToken);
StreamingWriteClientMessage clientMessage;
StreamingWriteServerMessage serverMessage;
- clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
+ clientMessage.mutable_init_request()->set_topic(server.GetTopicPath());
clientMessage.mutable_init_request()->set_message_group_id("test-message-group");
AssertSuccessfullStreamingOperation(stream->Write(clientMessage), stream, &clientMessage);
AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream);
@@ -272,7 +272,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
serverMessage.server_message_case(), serverMessage);
}
- void TestWriteWithRateLimiter(TPersQueueV1TestServerWithRateLimiter& server) {
+ void TestWriteWithRateLimiter(TPersQueueV1TestServerWithRateLimiter& server) {
const std::vector<TString> differentTopicPathsTypes = {
"account1/topic", // without folder
"account2/folder/topic", // with folder
@@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
};
const TString data = TString("12345") * 100;
for (const TString &topicPath : differentTopicPathsTypes) {
- server.CreateTopicWithQuota(topicPath);
+ server.CreateTopicWithQuota(topicPath);
auto driver = server.Server->AnnoyingClient->GetDriver();
@@ -293,22 +293,22 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
}
}
- Y_UNIT_TEST(TestWriteWithRateLimiterWithBlobsRateLimit) {
- TPersQueueV1TestServerWithRateLimiter server;
- server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::WRITTEN_BLOB_SIZE);
- TestWriteWithRateLimiter(server);
+ Y_UNIT_TEST(TestWriteWithRateLimiterWithBlobsRateLimit) {
+ TPersQueueV1TestServerWithRateLimiter server;
+ server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::WRITTEN_BLOB_SIZE);
+ TestWriteWithRateLimiter(server);
}
- Y_UNIT_TEST(TestWriteWithRateLimiterWithUserPayloadRateLimit) {
- TPersQueueV1TestServerWithRateLimiter server;
- server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE);
- TestWriteWithRateLimiter(server);
+ Y_UNIT_TEST(TestWriteWithRateLimiterWithUserPayloadRateLimit) {
+ TPersQueueV1TestServerWithRateLimiter server;
+ server.InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE);
+ TestWriteWithRateLimiter(server);
}
- void TestRateLimiterLimitsWrite(TPersQueueV1TestServerWithRateLimiter& server) {
+ void TestRateLimiterLimitsWrite(TPersQueueV1TestServerWithRateLimiter& server) {
const TString topicPath = "account/topic";
- server.CreateTopicWithQuota(topicPath, true, 100.0);
+ server.CreateTopicWithQuota(topicPath, true, 100.0);
const TString data = TString("123") * 100; // 300 bytes // 3 seconds
auto driver = server.Server->AnnoyingClient->GetDriver();
@@ -348,7 +348,7 @@ Y_UNIT_TEST_SUITE(TPersQueueCommonTest) {
// Check write time with quota
const TDuration writeTime = endWrite - startWrite;
// in new scheme cache rate limiting is turned off
- if (server.TenantModeEnabled()) {
+ if (server.TenantModeEnabled()) {
UNIT_ASSERT_GE_C(TDuration::Seconds(3), writeTime, "Write time: " << writeTime);
} else {
UNIT_ASSERT_GE_C(writeTime, TDuration::Seconds(3), "Write time: " << writeTime);
diff --git a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp
index 23e45478d1..8e209dbd9d 100644
--- a/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp
+++ b/ydb/services/persqueue_v1/persqueue_new_schemecache_ut.cpp
@@ -24,13 +24,13 @@
#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/ut_utils/data_plane_helpers.h>
namespace {
- const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1";
- const static TString DEFAULT_TOPIC_PATH = "/Root/PQ/rt3.dc1--topic1";
-
- const static TString SHORT_TOPIC_NAME = "topic1";
+ const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1";
+ const static TString DEFAULT_TOPIC_PATH = "/Root/PQ/rt3.dc1--topic1";
+
+ const static TString SHORT_TOPIC_NAME = "topic1";
}
-
+
namespace NKikimr::NPersQueueTests {
using namespace Tests;
@@ -40,7 +40,7 @@ namespace NKikimr::NPersQueueTests {
using namespace NThreading;
using namespace NNetClassifier;
using namespace NYdb::NPersQueue;
- using namespace NPersQueue;
+ using namespace NPersQueue;
NJson::TJsonValue GetCountersNewSchemeCache(ui16 port, const TString& counters, const TString& subsystem, const TString& topicPath) {
TString escapedPath = "%2F" + JoinStrings(SplitString(topicPath, "/"), "%2F");
@@ -69,28 +69,28 @@ namespace NKikimr::NPersQueueTests {
Y_UNIT_TEST_SUITE(TPersQueueNewSchemeCacheTest) {
void PrepareForGrpcNoDC(TFlatMsgBusPQClient& annoyingClient) {
- annoyingClient.SetNoConfigMode();
- annoyingClient.FullInit();
+ annoyingClient.SetNoConfigMode();
+ annoyingClient.FullInit();
annoyingClient.InitUserRegistry();
annoyingClient.MkDir("/Root", "account1");
annoyingClient.MkDir("/Root/PQ", "account1");
- annoyingClient.CreateTopicNoLegacy(DEFAULT_TOPIC_PATH, 5, false);
- annoyingClient.CreateTopicNoLegacy("/Root/PQ/account1/topic1", 5, false);
+ annoyingClient.CreateTopicNoLegacy(DEFAULT_TOPIC_PATH, 5, false);
+ annoyingClient.CreateTopicNoLegacy("/Root/PQ/account1/topic1", 5, false);
annoyingClient.CreateTopicNoLegacy("/Root/account2/topic2", 5);
}
Y_UNIT_TEST(CheckGrpcWriteNoDC) {
- TTestServer server(false);
- server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true);
+ TTestServer server(false);
+ server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true);
- server.StartServer();
- server.EnableLogs({
- NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE,
- NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE}
- );
- PrepareForGrpcNoDC(*server.AnnoyingClient);
+ server.StartServer();
+ server.EnableLogs({
+ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE,
+ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PQ_METACACHE}
+ );
+ PrepareForGrpcNoDC(*server.AnnoyingClient);
- TPQDataWriter writer("source1", server, DEFAULT_TOPIC_PATH);
+ TPQDataWriter writer("source1", server, DEFAULT_TOPIC_PATH);
writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
writer.Write("/Root/PQ/account1/topic1", {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
@@ -98,7 +98,7 @@ namespace NKikimr::NPersQueueTests {
NACLib::TDiffACL acl;
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN);
server.AnnoyingClient->ModifyACL("/Root/account2", "topic2", acl.SerializeAsString());
- server.AnnoyingClient->ModifyACL("/Root/PQ/account1", "topic1", acl.SerializeAsString());
+ server.AnnoyingClient->ModifyACL("/Root/PQ/account1", "topic1", acl.SerializeAsString());
WaitACLModification();
writer.Write("/Root/account2/topic2", {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
@@ -109,17 +109,17 @@ namespace NKikimr::NPersQueueTests {
}
Y_UNIT_TEST(CheckGrpcReadNoDC) {
- TTestServer server(false);
- server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true);
- server.StartServer();
- server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE});
- PrepareForGrpcNoDC(*server.AnnoyingClient);
- NYdb::TDriverConfig driverCfg;
+ TTestServer server(false);
+ server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(true);
+ server.StartServer();
+ server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::TX_PROXY_SCHEME_CACHE});
+ PrepareForGrpcNoDC(*server.AnnoyingClient);
+ NYdb::TDriverConfig driverCfg;
driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort).SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)).SetDatabase("/Root");
- auto ydbDriver = MakeHolder<NYdb::TDriver>(driverCfg);
- auto persQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*ydbDriver);
+ auto ydbDriver = MakeHolder<NYdb::TDriver>(driverCfg);
+ auto persQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*ydbDriver);
{
auto res = persQueueClient->AddReadRule("/Root/account2/topic2", TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName("user1")));
@@ -165,25 +165,25 @@ namespace NKikimr::NPersQueueTests {
Y_UNIT_TEST_SUITE(TPersqueueDataPlaneTestSuite) {
- Y_UNIT_TEST(WriteSession) {
- TPersQueueV1TestServer server(true);
+ Y_UNIT_TEST(WriteSession) {
+ TPersQueueV1TestServer server(true);
TString topic = "/Root/account1/write_topic";
TString consumer = "consumer_aba";
{
- auto res = server.PersQueueClient->CreateTopic(topic);
+ auto res = server.PersQueueClient->CreateTopic(topic);
res.Wait();
UNIT_ASSERT(res.GetValue().IsSuccess());
}
{
- auto res = server.PersQueueClient->AddReadRule(topic, TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName(consumer)));
+ auto res = server.PersQueueClient->AddReadRule(topic, TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName(consumer)));
res.Wait();
UNIT_ASSERT(res.GetValue().IsSuccess());
}
{
- auto writer = server.PersQueueClient->CreateSimpleBlockingWriteSession(TWriteSessionSettings()
+ auto writer = server.PersQueueClient->CreateSimpleBlockingWriteSession(TWriteSessionSettings()
.Path(topic).MessageGroupId("my_group_1")
.ClusterDiscoveryMode(EClusterDiscoveryMode::Off)
.RetryPolicy(IRetryPolicy::GetNoRetryPolicy()));
@@ -192,7 +192,7 @@ namespace NKikimr::NPersQueueTests {
writer->Close();
}
{
- auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName("non_existing")
+ auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName("non_existing")
.AppendTopics(topic).DisableClusterDiscovery(true)
.RetryPolicy(IRetryPolicy::GetNoRetryPolicy()));
@@ -209,7 +209,7 @@ namespace NKikimr::NPersQueueTests {
UNIT_ASSERT(std::get_if<TSessionClosedEvent>(&*event));
}
{
- auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName(consumer)
+ auto reader = server.PersQueueClient->CreateReadSession(TReadSessionSettings().ConsumerName(consumer)
.AppendTopics(topic).DisableClusterDiscovery(true)
.RetryPolicy(IRetryPolicy::GetNoRetryPolicy()));
@@ -229,18 +229,18 @@ namespace NKikimr::NPersQueueTests {
}
Y_UNIT_TEST_SUITE(TPersqueueControlPlaneTestSuite) {
- Y_UNIT_TEST(SetupReadLockSessionWithDatabase) {
- TPersQueueV1TestServer server;
+ Y_UNIT_TEST(SetupReadLockSessionWithDatabase) {
+ TPersQueueV1TestServer server;
{
- auto res = server.PersQueueClient->AddReadRule("/Root/acc/topic1", TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName("user")));
+ auto res = server.PersQueueClient->AddReadRule("/Root/acc/topic1", TAddReadRuleSettings().ReadRule(TReadRuleSettings().ConsumerName("user")));
res.Wait();
Cerr << "ADD RESULT " << res.GetValue().GetIssues().ToString() << "\n";
UNIT_ASSERT(res.GetValue().IsSuccess());
}
- auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel);
+ auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel);
grpc::ClientContext grpcContext;
grpcContext.AddMetadata("x-ydb-database", "/Root/acc");
auto readStream = stub->MigrationStreamingRead(&grpcContext);
@@ -265,10 +265,10 @@ namespace NKikimr::NPersQueueTests {
}
}
- Y_UNIT_TEST(SetupWriteLockSessionWithDatabase) {
- TPersQueueV1TestServer server;
-
- auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel);
+ Y_UNIT_TEST(SetupWriteLockSessionWithDatabase) {
+ TPersQueueV1TestServer server;
+
+ auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel);
grpc::ClientContext grpcContext;
grpcContext.AddMetadata("x-ydb-database", "/Root/acc");
@@ -290,19 +290,19 @@ namespace NKikimr::NPersQueueTests {
}
}
- Y_UNIT_TEST(TestAddRemoveReadRule) {
- TPersQueueV1TestServer server;
- SET_LOCALS;
+ Y_UNIT_TEST(TestAddRemoveReadRule) {
+ TPersQueueV1TestServer server;
+ SET_LOCALS;
+
+ pqClient->CreateConsumer("goodUser");
- pqClient->CreateConsumer("goodUser");
+ TString path = server.GetTopicPath();
- TString path = server.GetTopicPath();
-
Ydb::PersQueue::V1::AddReadRuleRequest addRuleRequest;
Ydb::PersQueue::V1::AddReadRuleResponse addRuleResponse;
addRuleRequest.set_path(path);
- auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel);
+ auto stub = Ydb::PersQueue::V1::PersQueueService::NewStub(server.InsecureChannel);
{
grpc::ClientContext grpcContext;
grpcContext.AddMetadata("x-ydb-database", "/Root/acc");
diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp
index f795097918..3bc9b21728 100644
--- a/ydb/services/persqueue_v1/persqueue_ut.cpp
+++ b/ydb/services/persqueue_v1/persqueue_ut.cpp
@@ -78,27 +78,27 @@ static TString FormNetData() {
"::1/128\tVLA\n";
}
-namespace {
- const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1";
- const static TString SHORT_TOPIC_NAME = "topic1";
-}
-
-#define MAKE_INSECURE_STUB \
- std::shared_ptr<grpc::Channel> Channel_; \
- std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; \
- \
- { \
- Channel_ = grpc::CreateChannel( \
- "localhost:" + ToString(server.Server->GrpcPort), \
- grpc::InsecureChannelCredentials() \
- ); \
- StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); \
- } \
- grpc::ClientContext rcontext;
-
-
-
- Y_UNIT_TEST_SUITE(TPersQueueTest) {
+namespace {
+ const static TString DEFAULT_TOPIC_NAME = "rt3.dc1--topic1";
+ const static TString SHORT_TOPIC_NAME = "topic1";
+}
+
+#define MAKE_INSECURE_STUB \
+ std::shared_ptr<grpc::Channel> Channel_; \
+ std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_; \
+ \
+ { \
+ Channel_ = grpc::CreateChannel( \
+ "localhost:" + ToString(server.Server->GrpcPort), \
+ grpc::InsecureChannelCredentials() \
+ ); \
+ StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_); \
+ } \
+ grpc::ClientContext rcontext;
+
+
+
+ Y_UNIT_TEST_SUITE(TPersQueueTest) {
Y_UNIT_TEST(AllEqual) {
using NGRpcProxy::V1::AllEqual;
@@ -115,10 +115,10 @@ namespace {
UNIT_ASSERT(!AllEqual(1, 1, 1, 1, 1, 0));
}
- Y_UNIT_TEST(SetupLockSession2) {
- TPersQueueV1TestServer server;
- SET_LOCALS;
- const TString topicPath = server.GetTopicPathMultipleDC();
+ Y_UNIT_TEST(SetupLockSession2) {
+ TPersQueueV1TestServer server;
+ SET_LOCALS;
+ const TString topicPath = server.GetTopicPathMultipleDC();
auto driver = server.Server->AnnoyingClient->GetDriver();
@@ -130,7 +130,7 @@ namespace {
auto msg = reader->GetEvent(true, 1);
UNIT_ASSERT(msg);
- Cerr << "Got message: " << NYdb::NPersQueue::DebugString(*msg) << "\n";
+ Cerr << "Got message: " << NYdb::NPersQueue::DebugString(*msg) << "\n";
auto ev = std::get_if<NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent>(&*msg);
@@ -144,8 +144,8 @@ namespace {
auto wait = reader->WaitEvent();
UNIT_ASSERT(!wait.Wait(TDuration::Seconds(1)));
- pqClient->AlterTopicNoLegacy("/Root/PQ/rt3.dc2--acc--topic2dc", 2);
- Cerr << "======Alter topic done\n";
+ pqClient->AlterTopicNoLegacy("/Root/PQ/rt3.dc2--acc--topic2dc", 2);
+ Cerr << "======Alter topic done\n";
UNIT_ASSERT(wait.Wait(TDuration::Seconds(5)));
auto msg = reader->GetEvent(true, 1);
@@ -163,10 +163,10 @@ namespace {
}
- Y_UNIT_TEST(SetupLockSession) {
- TPersQueueV1TestServer server;
- SET_LOCALS;
- MAKE_INSECURE_STUB;
+ Y_UNIT_TEST(SetupLockSession) {
+ TPersQueueV1TestServer server;
+ SET_LOCALS;
+ MAKE_INSECURE_STUB;
auto readStream = StubP_->MigrationStreamingRead(&rcontext);
UNIT_ASSERT(readStream);
@@ -178,14 +178,14 @@ namespace {
req.mutable_init_request()->add_topics_read_settings()->set_topic("acc/topic1");
req.mutable_init_request()->set_consumer("user");
- req.mutable_init_request()->set_read_only_original(true);
+ req.mutable_init_request()->set_read_only_original(true);
req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(1);
if (!readStream->Write(req)) {
ythrow yexception() << "write fail";
}
UNIT_ASSERT(readStream->Read(&resp));
- Cerr << "===Got response: " << resp.ShortDebugString() << Endl;
+ Cerr << "===Got response: " << resp.ShortDebugString() << Endl;
UNIT_ASSERT(resp.response_case() == MigrationStreamingReadServerMessage::kInitResponse);
//send some reads
req.Clear();
@@ -198,19 +198,19 @@ namespace {
}
{
- Sleep(TDuration::Seconds(10));
+ Sleep(TDuration::Seconds(10));
ReadInfoRequest request;
ReadInfoResponse response;
request.mutable_consumer()->set_path("user");
- request.set_get_only_original(true);
+ request.set_get_only_original(true);
request.add_topics()->set_path("acc/topic1");
grpc::ClientContext rcontext;
auto status = StubP_->GetReadSessionsInfo(&rcontext, request, &response);
UNIT_ASSERT(status.ok());
ReadInfoResult res;
response.operation().result().UnpackTo(&res);
- Cerr << "Read info response: " << response << Endl << res << Endl;
- UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1);
+ Cerr << "Read info response: " << response << Endl << res << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1);
UNIT_ASSERT(res.topics(0).status() == Ydb::StatusIds::SUCCESS);
}
ui64 assignId = 0;
@@ -279,7 +279,7 @@ namespace {
}
- pqClient->AlterTopic("rt3.dc1--acc--topic1", 10);
+ pqClient->AlterTopic("rt3.dc1--acc--topic1", 10);
{
ReadInfoRequest request;
ReadInfoResponse response;
@@ -291,10 +291,10 @@ namespace {
UNIT_ASSERT(status.ok());
ReadInfoResult res;
response.operation().result().UnpackTo(&res);
- Cerr << "Get read session info response: " << response << "\n" << res << "\n";
+ Cerr << "Get read session info response: " << response << "\n" << res << "\n";
// UNIT_ASSERT(res.sessions_size() == 1);
- UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1);
- UNIT_ASSERT_VALUES_EQUAL(res.topics(0).partitions_size(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(res.topics_size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(res.topics(0).partitions_size(), 10);
}
{
@@ -305,39 +305,39 @@ namespace {
request.add_topics()->set_path("acc/topic1");
grpc::ClientContext rcontext;
- pqClient->MarkNodeInHive(runtime, 0, false);
- pqClient->MarkNodeInHive(runtime, 1, false);
+ pqClient->MarkNodeInHive(runtime, 0, false);
+ pqClient->MarkNodeInHive(runtime, 1, false);
- pqClient->RestartBalancerTablet(runtime, "rt3.dc1--acc--topic1");
+ pqClient->RestartBalancerTablet(runtime, "rt3.dc1--acc--topic1");
auto status = StubP_->GetReadSessionsInfo(&rcontext, request, &response);
UNIT_ASSERT(status.ok());
ReadInfoResult res;
response.operation().result().UnpackTo(&res);
- Cerr << "Read sessions info response: " << response << "\nResult: " << res << "\n";
+ Cerr << "Read sessions info response: " << response << "\nResult: " << res << "\n";
UNIT_ASSERT(res.topics().size() == 1);
- UNIT_ASSERT(res.topics(0).partitions(0).status() == Ydb::StatusIds::UNAVAILABLE);
+ UNIT_ASSERT(res.topics(0).partitions(0).status() == Ydb::StatusIds::UNAVAILABLE);
}
}
void SetupWriteSessionImpl(bool rr) {
- NPersQueue::TTestServer server{PQSettings(0, 2, rr), false};
- server.ServerSettings.SetEnableSystemViews(false);
- server.StartServer();
+ NPersQueue::TTestServer server{PQSettings(0, 2, rr), false};
+ server.ServerSettings.SetEnableSystemViews(false);
+ server.StartServer();
- server.EnableLogs({ NKikimrServices::PERSQUEUE });
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
+ server.EnableLogs({ NKikimrServices::PERSQUEUE });
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- TPQDataWriter writer("source", server);
+ TPQDataWriter writer("source", server);
- ui32 p = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"});
+ ui32 p = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"});
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 15);
+ server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 15);
- ui32 pp = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"});
+ ui32 pp = writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"});
UNIT_ASSERT_VALUES_EQUAL(p, pp);
- writer.Write(SHORT_TOPIC_NAME, {"1", "2", "3", "4", "5"});
+ writer.Write(SHORT_TOPIC_NAME, {"1", "2", "3", "4", "5"});
writer.Write("topic2", {"valuevaluevalue1"}, true);
@@ -377,9 +377,9 @@ namespace {
Y_UNIT_TEST(StoreNoMoreThanXSourceIDs) {
ui16 X = 4;
ui64 SOURCEID_COUNT_DELETE_BATCH_SIZE = 100;
- NPersQueue::TTestServer server;
- server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::PQ_WRITE_PROXY });
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {}, {}, {}, X, 86400);
+ NPersQueue::TTestServer server;
+ server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::PQ_WRITE_PROXY });
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {}, {}, {}, X, 86400);
auto driver = server.AnnoyingClient->GetDriver();
@@ -429,8 +429,8 @@ namespace {
}
Y_UNIT_TEST(EachMessageGetsExactlyOneAcknowledgementInCorrectOrder) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic("rt3.dc1--topic", 1);
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic("rt3.dc1--topic", 1);
auto driver = server.AnnoyingClient->GetDriver();
@@ -449,24 +449,24 @@ namespace {
UNIT_ASSERT(res);
}
- Y_UNIT_TEST(SetupWriteSessionOnDisabledCluster) {
- TPersQueueV1TestServer server;
- SET_LOCALS;
+ Y_UNIT_TEST(SetupWriteSessionOnDisabledCluster) {
+ TPersQueueV1TestServer server;
+ SET_LOCALS;
- TPQDataWriter writer("source", *server.Server);
+ TPQDataWriter writer("source", *server.Server);
- pqClient->DisableDC();
+ pqClient->DisableDC();
Sleep(TDuration::Seconds(5));
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true);
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true);
}
Y_UNIT_TEST(CloseActiveWriteSessionOnClusterDisable) {
- NPersQueue::TTestServer server;
+ NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
+ server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
TPQDataWriter writer2("source", server);
@@ -480,7 +480,7 @@ namespace {
Cerr << DebugString(*msg) << "\n";
- server.AnnoyingClient->DisableDC();
+ server.AnnoyingClient->DisableDC();
UNIT_ASSERT(writer->WaitEvent().Wait(TDuration::Seconds(30)));
msg = writer->GetEvent(true);
@@ -498,11 +498,11 @@ namespace {
}
Y_UNIT_TEST(BadSids) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
+ server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
TPQDataWriter writer2("source", server);
- TString topic = SHORT_TOPIC_NAME;
+ TString topic = SHORT_TOPIC_NAME;
auto driver = server.AnnoyingClient->GetDriver();
@@ -518,26 +518,26 @@ namespace {
}
Y_UNIT_TEST(ReadFromSeveralPartitions) {
- NPersQueue::TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_METACACHE });
+ NPersQueue::TTestServer server;
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_METACACHE });
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- TPQDataWriter writer("source1", server);
- Cerr << "===Writer started\n";
+ TPQDataWriter writer("source1", server);
+ Cerr << "===Writer started\n";
std::shared_ptr<grpc::Channel> Channel_;
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_;
- Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_);
//Write some data
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"});
- TPQDataWriter writer2("source2", server);
- writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"});
- Cerr << "===Writer - writes done\n";
+ TPQDataWriter writer2("source2", server);
+ writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"});
+ Cerr << "===Writer - writes done\n";
grpc::ClientContext rcontext;
auto readStream = StubP_->MigrationStreamingRead(&rcontext);
@@ -548,20 +548,20 @@ namespace {
MigrationStreamingReadClientMessage req;
MigrationStreamingReadServerMessage resp;
- req.mutable_init_request()->add_topics_read_settings()->set_topic(SHORT_TOPIC_NAME);
+ req.mutable_init_request()->add_topics_read_settings()->set_topic(SHORT_TOPIC_NAME);
req.mutable_init_request()->set_consumer("user");
- req.mutable_init_request()->set_read_only_original(true);
-
+ req.mutable_init_request()->set_read_only_original(true);
+
req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(1000);
if (!readStream->Write(req)) {
ythrow yexception() << "write fail";
}
- Cerr << "===Try to get read response\n";
-
+ Cerr << "===Try to get read response\n";
+
UNIT_ASSERT(readStream->Read(&resp));
- Cerr << "Read server response: " << resp.ShortDebugString() << Endl;
+ Cerr << "Read server response: " << resp.ShortDebugString() << Endl;
UNIT_ASSERT(resp.response_case() == MigrationStreamingReadServerMessage::kInitResponse);
//send some reads
@@ -584,7 +584,7 @@ namespace {
if (resp.response_case() == MigrationStreamingReadServerMessage::kAssigned) {
auto assignId = resp.assigned().assign_id();
MigrationStreamingReadClientMessage req;
- req.mutable_start_read()->mutable_topic()->set_path(SHORT_TOPIC_NAME);
+ req.mutable_start_read()->mutable_topic()->set_path(SHORT_TOPIC_NAME);
req.mutable_start_read()->set_cluster("dc1");
req.mutable_start_read()->set_assign_id(assignId);
UNIT_ASSERT(readStream->Write(req));
@@ -598,26 +598,26 @@ namespace {
void SetupReadSessionTest() {
- NPersQueue::TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ NPersQueue::TTestServer server;
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
server.AnnoyingClient->CreateTopicNoLegacy("rt3.dc2--topic1", 2, true, false);
- TPQDataWriter writer("source1", server);
+ TPQDataWriter writer("source1", server);
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue0"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue3"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue4"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue5"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue6"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue7"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue8"});
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue9"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue0"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue2"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue3"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue4"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue5"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue6"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue7"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue8"});
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue9"});
- writer.Read(SHORT_TOPIC_NAME, "user1", "", false, false);
+ writer.Read(SHORT_TOPIC_NAME, "user1", "", false, false);
}
Y_UNIT_TEST(SetupReadSession) {
@@ -626,17 +626,17 @@ namespace {
Y_UNIT_TEST(WriteExisting) {
- NPersQueue::TTestServer server;
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ NPersQueue::TTestServer server;
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
{
THolder<NMsgBusProxy::TBusPersQueue> request = TRequestDescribePQ().GetRequest({});
NKikimrClient::TResponse response;
- auto channel = grpc::CreateChannel("localhost:"+ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ auto channel = grpc::CreateChannel("localhost:"+ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
auto stub(NKikimrClient::TGRpcServer::NewStub(channel));
grpc::ClientContext context;
auto status = stub->PersQueueRequest(&context, request->Record, &response);
@@ -644,177 +644,177 @@ namespace {
UNIT_ASSERT(status.ok());
}
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1", "", ETransport::GRpc);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, "valuevaluevalue1", "", ETransport::GRpc);
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1", "", ETransport::GRpc);
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, "valuevaluevalue1", "", ETransport::GRpc);
}
Y_UNIT_TEST(WriteExistingBigValue) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2, 8*1024*1024, 86400, 100000);
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2, 8*1024*1024, 86400, 100000);
TInstant now(Now());
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, TString(1000000, 'a'));
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, TString(1, 'a'));
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, TString(1000000, 'a'));
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 2, TString(1, 'a'));
UNIT_ASSERT(TInstant::Now() - now > TDuration::MilliSeconds(5990)); //speed limit is 200kb/s and burst is 200kb, so to write 1mb it will take at least 4 seconds
}
Y_UNIT_TEST(WriteEmptyData) {
- NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)};
+ NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)};
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.EnableLogs({ NKikimrServices::PERSQUEUE });
+ server.EnableLogs({ NKikimrServices::PERSQUEUE });
// empty data and sourceId
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_OK);
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR);
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "a", 1, "a", "", ETransport::MsgBus, NMsgBusProxy::MSTATUS_OK);
}
Y_UNIT_TEST(WriteNonExistingPartition) {
- NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)};
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ NPersQueue::TTestServer server{PQSettings(0).SetDomainName("Root").SetNodeCount(2)};
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 100500, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
+ server.AnnoyingClient->WriteToPQ(
+ DEFAULT_TOPIC_NAME, 100500, "abacaba", 1, "valuevaluevalue1", "",
+ ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
+ );
}
Y_UNIT_TEST(WriteNonExistingTopic) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME + "000", 1, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
+ server.AnnoyingClient->WriteToPQ(
+ DEFAULT_TOPIC_NAME + "000", 1, "abacaba", 1, "valuevaluevalue1", "",
+ ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
+ );
}
Y_UNIT_TEST(SchemeshardRestart) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(1));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- TString secondTopic = "rt3.dc1--topic2";
- server.AnnoyingClient->CreateTopic(secondTopic, 2);
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(1));
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ TString secondTopic = "rt3.dc1--topic2";
+ server.AnnoyingClient->CreateTopic(secondTopic, 2);
// force topic1 into cache and establish pipe from cache to schemeshard
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1");
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 1, "abacaba", 1, "valuevaluevalue1");
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD,
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD,
NKikimrServices::PERSQUEUE,
NKikimrServices::PQ_METACACHE });
- server.AnnoyingClient->RestartSchemeshard(server.CleverServer->GetRuntime());
+ server.AnnoyingClient->RestartSchemeshard(server.CleverServer->GetRuntime());
- server.AnnoyingClient->WriteToPQ(secondTopic, 1, "abacaba", 1, "valuevaluevalue1");
+ server.AnnoyingClient->WriteToPQ(secondTopic, 1, "abacaba", 1, "valuevaluevalue1");
}
Y_UNIT_TEST(WriteAfterAlter) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
+ server.AnnoyingClient->WriteToPQ(
+ DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1", "",
+ ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
+ );
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD,
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD,
NKikimrServices::PERSQUEUE,
NKikimrServices::PQ_METACACHE });
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 10);
+ server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 10);
Sleep(TDuration::Seconds(1));
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
- server.AnnoyingClient->WriteToPQ(
- DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1", "",
- ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
- );
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
+ server.AnnoyingClient->WriteToPQ(
+ DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1", "",
+ ETransport::MsgBus, NMsgBusProxy::MSTATUS_ERROR, NMsgBusProxy::MSTATUS_ERROR
+ );
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20);
+ server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20);
Sleep(TDuration::Seconds(1));
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1");
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 15, "abacaba", 1, "valuevaluevalue1");
}
Y_UNIT_TEST(Delete) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE});
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE});
// Delete non-existing
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC);
+ server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
// Delete existing
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
+ server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
// Double delete - "What Is Dead May Never Die"
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC);
+ server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::UNKNOWN_TOPIC);
// Resurrect deleted topic
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME);
}
Y_UNIT_TEST(BigRead) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400, 20000000, "user1", 2000000);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
TString value(1024*1024, 'x');
for (ui32 i = 0; i < 32; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value);
// trying to read small PQ messages in a big messagebus event
- auto info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb
+ auto info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb
UNIT_ASSERT_VALUES_EQUAL(info.BlobsFromDisk, 0);
UNIT_ASSERT_VALUES_EQUAL(info.BlobsFromCache, 4);
TInstant now(TInstant::Now());
- info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb
+ info = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 32, "user1"}, 23, "", NMsgBusProxy::MSTATUS_OK); //will read 21mb
TDuration dur = TInstant::Now() - now;
UNIT_ASSERT_C(dur > TDuration::Seconds(7) && dur < TDuration::Seconds(20), "dur = " << dur); //speed limit is 2000kb/s and burst is 2000kb, so to read 24mb it will take at least 11 seconds
- server.AnnoyingClient->GetPartStatus({}, 1, true);
+ server.AnnoyingClient->GetPartStatus({}, 1, true);
}
// expects that L2 size is 32Mb
Y_UNIT_TEST(Cache) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400);
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 8*1024*1024, 86400);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
TString value(1024*1024, 'x');
for (ui32 i = 0; i < 32; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, value);
- auto info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16);
- auto info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16);
+ auto info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16);
+ auto info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16);
UNIT_ASSERT_VALUES_EQUAL(info0.BlobsFromCache, 3);
UNIT_ASSERT_VALUES_EQUAL(info16.BlobsFromCache, 2);
UNIT_ASSERT_VALUES_EQUAL(info0.BlobsFromDisk + info16.BlobsFromDisk, 0);
for (ui32 i = 0; i < 8; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 32+i}, value);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 32+i}, value);
- info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16);
- info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16);
+ info0 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 16, "user1"}, 16);
+ info16 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 16, 16, "user1"}, 16);
ui32 fromDisk = info0.BlobsFromDisk + info16.BlobsFromDisk;
ui32 fromCache = info0.BlobsFromCache + info16.BlobsFromCache;
@@ -825,10 +825,10 @@ namespace {
}
Y_UNIT_TEST(CacheHead) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400);
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
ui64 seqNo = 0;
for (ui32 blobSizeKB = 256; blobSizeKB < 4096; blobSizeKB *= 2) {
@@ -841,10 +841,10 @@ namespace {
ui64 offset = seqNo;
TString value(blobSize, 'a');
for (ui32 i = 0; i < count; ++i)
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", seqNo++}, value);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", seqNo++}, value);
- auto info_half1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half);
- auto info_half2 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half);
+ auto info_half1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half);
+ auto info_half2 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, offset, half, "user1"}, half);
UNIT_ASSERT(info_half1.BlobsFromCache > 0);
UNIT_ASSERT(info_half2.BlobsFromCache > 0);
@@ -854,28 +854,28 @@ namespace {
}
Y_UNIT_TEST(SameOffset) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400);
- TString secondTopic = DEFAULT_TOPIC_NAME + "2";
- server.AnnoyingClient->CreateTopic(secondTopic, 1, 6*1024*1024, 86400);
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1, 6*1024*1024, 86400);
+ TString secondTopic = DEFAULT_TOPIC_NAME + "2";
+ server.AnnoyingClient->CreateTopic(secondTopic, 1, 6*1024*1024, 86400);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
ui32 valueSize = 128;
TString value1(valueSize, 'a');
TString value2(valueSize, 'b');
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 0}, value1);
- server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", 0}, value2);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", 0}, value1);
+ server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", 0}, value2);
// avoid reading from head
TString mb(1024*1024, 'x');
for (ui32 i = 1; i < 16; ++i) {
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, mb);
- server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", i}, mb);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 0, "source1", i}, mb);
+ server.AnnoyingClient->WriteToPQ({secondTopic, 0, "source1", i}, mb);
}
- auto info1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 1, "user1"}, 1);
- auto info2 = server.AnnoyingClient->ReadFromPQ({secondTopic, 0, 0, 1, "user1"}, 1);
+ auto info1 = server.AnnoyingClient->ReadFromPQ({DEFAULT_TOPIC_NAME, 0, 0, 1, "user1"}, 1);
+ auto info2 = server.AnnoyingClient->ReadFromPQ({secondTopic, 0, 0, 1, "user1"}, 1);
UNIT_ASSERT_VALUES_EQUAL(info1.BlobsFromCache, 1);
UNIT_ASSERT_VALUES_EQUAL(info2.BlobsFromCache, 1);
@@ -889,34 +889,34 @@ namespace {
Y_UNIT_TEST(FetchRequest) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
- TString secondTopic = DEFAULT_TOPIC_NAME + "2";
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root"));
+ TString secondTopic = DEFAULT_TOPIC_NAME + "2";
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.AnnoyingClient->CreateTopic(secondTopic, 10);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
+ server.AnnoyingClient->CreateTopic(secondTopic, 10);
ui32 valueSize = 128;
TString value1(valueSize, 'a');
TString value2(valueSize, 'b');
- server.AnnoyingClient->WriteToPQ({secondTopic, 5, "source1", 0}, value2);
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 0}, value1);
- server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 1}, value2);
+ server.AnnoyingClient->WriteToPQ({secondTopic, 5, "source1", 0}, value2);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 0}, value1);
+ server.AnnoyingClient->WriteToPQ({DEFAULT_TOPIC_NAME, 1, "source1", 1}, value2);
- server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
+ server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE });
TInstant tm(TInstant::Now());
- server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 400, 1000000);
+ server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 400, 1000000);
UNIT_ASSERT((TInstant::Now() - tm).Seconds() < 1);
tm = TInstant::Now();
- server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 1, 400}}, 400, 5000);
+ server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 1, 400}}, 400, 5000);
UNIT_ASSERT((TInstant::Now() - tm).Seconds() > 2);
- server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 1, 1000000);
- server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 500, 400},{secondTopic, 4, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400}}, 400, 1000000);
+ server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400},{DEFAULT_TOPIC_NAME, 3, 0, 400}}, 1, 1000000);
+ server.AnnoyingClient->FetchRequestPQ({{secondTopic, 5, 500, 400},{secondTopic, 4, 0, 400},{DEFAULT_TOPIC_NAME, 1, 0, 400}}, 400, 1000000);
}
Y_UNIT_TEST(Init) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
if (!true) {
- server.EnableLogs( {
+ server.EnableLogs( {
NKikimrServices::FLAT_TX_SCHEMESHARD,
NKikimrServices::TX_DATASHARD,
NKikimrServices::HIVE,
@@ -927,38 +927,38 @@ namespace {
NKikimrServices::PQ_METACACHE });
}
- server.AnnoyingClient->DescribeTopic({});
- server.AnnoyingClient->TestCase({}, 0, 0, true);
+ server.AnnoyingClient->DescribeTopic({});
+ server.AnnoyingClient->TestCase({}, 0, 0, true);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
- server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20);
- TString secondTopic = DEFAULT_TOPIC_NAME + "2";
- TString thirdTopic = DEFAULT_TOPIC_NAME + "3";
- server.AnnoyingClient->CreateTopic(secondTopic, 25);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10);
+ server.AnnoyingClient->AlterTopic(DEFAULT_TOPIC_NAME, 20);
+ TString secondTopic = DEFAULT_TOPIC_NAME + "2";
+ TString thirdTopic = DEFAULT_TOPIC_NAME + "3";
+ server.AnnoyingClient->CreateTopic(secondTopic, 25);
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 2, "valuevaluevalue2");
- server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacabae", 1, "valuevaluevalue3");
- server.AnnoyingClient->ReadFromPQ(DEFAULT_TOPIC_NAME, 5, 0, 10, 3);
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 1, "valuevaluevalue1");
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacaba", 2, "valuevaluevalue2");
+ server.AnnoyingClient->WriteToPQ(DEFAULT_TOPIC_NAME, 5, "abacabae", 1, "valuevaluevalue3");
+ server.AnnoyingClient->ReadFromPQ(DEFAULT_TOPIC_NAME, 5, 0, 10, 3);
- server.AnnoyingClient->SetClientOffsetPQ(DEFAULT_TOPIC_NAME, 5, 2);
+ server.AnnoyingClient->SetClientOffsetPQ(DEFAULT_TOPIC_NAME, 5, 2);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5}}}, 1, 1, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0}}}, 1, 0, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}}, 20, 1, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5, 5}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {111}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({}, 45, 1, true);
- server.AnnoyingClient->TestCase({{thirdTopic, {}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {thirdTopic, {}}}, 0, 0, false);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {secondTopic, {}}}, 45, 1, true);
- server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0, 3, 5}}, {secondTopic, {1, 4, 6, 8}}}, 7, 1, true);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5}}}, 1, 1, true);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0}}}, 1, 0, true);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}}, 20, 1, true);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {5, 5}}}, 0, 0, false);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {111}}}, 0, 0, false);
+ server.AnnoyingClient->TestCase({}, 45, 1, true);
+ server.AnnoyingClient->TestCase({{thirdTopic, {}}}, 0, 0, false);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {thirdTopic, {}}}, 0, 0, false);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {}}, {secondTopic, {}}}, 45, 1, true);
+ server.AnnoyingClient->TestCase({{DEFAULT_TOPIC_NAME, {0, 3, 5}}, {secondTopic, {1, 4, 6, 8}}}, 7, 1, true);
- server.AnnoyingClient->DescribeTopic({DEFAULT_TOPIC_NAME});
- server.AnnoyingClient->DescribeTopic({secondTopic});
- server.AnnoyingClient->DescribeTopic({secondTopic, DEFAULT_TOPIC_NAME});
- server.AnnoyingClient->DescribeTopic({});
- server.AnnoyingClient->DescribeTopic({thirdTopic}, true);
+ server.AnnoyingClient->DescribeTopic({DEFAULT_TOPIC_NAME});
+ server.AnnoyingClient->DescribeTopic({secondTopic});
+ server.AnnoyingClient->DescribeTopic({secondTopic, DEFAULT_TOPIC_NAME});
+ server.AnnoyingClient->DescribeTopic({});
+ server.AnnoyingClient->DescribeTopic({thirdTopic}, true);
}
void WaitResolveSuccess(TFlatMsgBusPQClient& annoyingClient, TString topic, ui32 numParts) {
@@ -985,28 +985,28 @@ namespace {
}
Y_UNIT_TEST(WhenDisableNodeAndCreateTopic_ThenAllPartitionsAreOnOtherNode) {
- NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
- server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::HIVE });
- TString unusedTopic = "rt3.dc1--unusedtopic";
- server.AnnoyingClient->CreateTopic(unusedTopic, 1);
- WaitResolveSuccess(*server.AnnoyingClient, unusedTopic, 1);
+ NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2));
+ server.EnableLogs({ NKikimrServices::PERSQUEUE, NKikimrServices::HIVE });
+ TString unusedTopic = "rt3.dc1--unusedtopic";
+ server.AnnoyingClient->CreateTopic(unusedTopic, 1);
+ WaitResolveSuccess(*server.AnnoyingClient, unusedTopic, 1);
// Act
// Disable node #0
- server.AnnoyingClient->MarkNodeInHive(server.CleverServer->GetRuntime(), 0, false);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
- WaitResolveSuccess(*server.AnnoyingClient, DEFAULT_TOPIC_NAME, 3);
+ server.AnnoyingClient->MarkNodeInHive(server.CleverServer->GetRuntime(), 0, false);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
+ WaitResolveSuccess(*server.AnnoyingClient, DEFAULT_TOPIC_NAME, 3);
// Assert that all partitions are on node #1
- const ui32 node1Id = server.CleverServer->GetRuntime()->GetNodeId(1);
+ const ui32 node1Id = server.CleverServer->GetRuntime()->GetNodeId(1);
UNIT_ASSERT_VALUES_EQUAL(
- server.AnnoyingClient->GetPartLocation({{DEFAULT_TOPIC_NAME, {0, 1}}}, 2, true),
+ server.AnnoyingClient->GetPartLocation({{DEFAULT_TOPIC_NAME, {0, 1}}}, 2, true),
TVector<ui32>({node1Id, node1Id})
);
}
- void PrepareForGrpc(NPersQueue::TTestServer& server) {
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
+ void PrepareForGrpc(NPersQueue::TTestServer& server) {
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2);
}
class TTestCredentialsProvider : public NYdb::ICredentialsProvider {
@@ -1064,25 +1064,25 @@ namespace {
Y_UNIT_TEST(CheckACLForGrpcWrite) {
- NPersQueue::TTestServer server(PQSettings(0, 1));
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
- PrepareForGrpc(server);
+ NPersQueue::TTestServer server(PQSettings(0, 1));
+ server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
+ PrepareForGrpc(server);
- TPQDataWriter writer("source1", server);
- TPQDataWriter writer2("source1", server);
+ TPQDataWriter writer("source1", server);
+ TPQDataWriter writer2("source1", server);
- server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
+ server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, TString()); // Fail if user set empty token
- writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, TString()); // Fail if user set empty token
+ writer.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "topic1@" BUILTIN_ACL_DOMAIN);
NACLib::TDiffACL acl;
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, "topic1@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->ModifyACL("/Root/PQ", DEFAULT_TOPIC_NAME, acl.SerializeAsString());
+ server.AnnoyingClient->ModifyACL("/Root/PQ", DEFAULT_TOPIC_NAME, acl.SerializeAsString());
WaitACLModification();
- writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
- writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "invalid_ticket");
+ writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, false, "topic1@" BUILTIN_ACL_DOMAIN);
+ writer2.Write(SHORT_TOPIC_NAME, {"valuevaluevalue1"}, true, "invalid_ticket");
auto driver = server.AnnoyingClient->GetDriver();
@@ -1136,67 +1136,67 @@ namespace {
Y_UNIT_TEST(CheckACLForGrpcRead) {
- NPersQueue::TTestServer server(PQSettings(0, 1));
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO);
- TString topic2 = DEFAULT_TOPIC_NAME + "2";
- TString shortTopic2Name = "topic12";
- PrepareForGrpc(server);
-
- server.AnnoyingClient->CreateTopic(topic2, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {"user1", "user2"});
- server.WaitInit(shortTopic2Name);
- server.AnnoyingClient->CreateConsumer("user1");
- server.AnnoyingClient->CreateConsumer("user2");
- server.AnnoyingClient->CreateConsumer("user5");
- server.AnnoyingClient->GrantConsumerAccess("user1", "user2@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->GrantConsumerAccess("user1", "user3@" BUILTIN_ACL_DOMAIN);
+ NPersQueue::TTestServer server(PQSettings(0, 1));
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_INFO);
+ TString topic2 = DEFAULT_TOPIC_NAME + "2";
+ TString shortTopic2Name = "topic12";
+ PrepareForGrpc(server);
+
+ server.AnnoyingClient->CreateTopic(topic2, 1, 8*1024*1024, 86400, 20000000, "", 200000000, {"user1", "user2"});
+ server.WaitInit(shortTopic2Name);
+ server.AnnoyingClient->CreateConsumer("user1");
+ server.AnnoyingClient->CreateConsumer("user2");
+ server.AnnoyingClient->CreateConsumer("user5");
+ server.AnnoyingClient->GrantConsumerAccess("user1", "user2@" BUILTIN_ACL_DOMAIN);
+ server.AnnoyingClient->GrantConsumerAccess("user1", "user3@" BUILTIN_ACL_DOMAIN);
server.AnnoyingClient->GrantConsumerAccess("user1", "1@" BUILTIN_ACL_DOMAIN);
server.AnnoyingClient->GrantConsumerAccess("user2", "2@" BUILTIN_ACL_DOMAIN);
server.AnnoyingClient->GrantConsumerAccess("user5", "1@" BUILTIN_ACL_DOMAIN);
server.AnnoyingClient->GrantConsumerAccess("user5", "2@" BUILTIN_ACL_DOMAIN);
- Cerr << "=== Create writer\n";
- TPQDataWriter writer("source1", server);
+ Cerr << "=== Create writer\n";
+ TPQDataWriter writer("source1", server);
- server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
+ server.CleverServer->GetRuntime()->GetAppData().PQConfig.SetRequireCredentialsInNewProtocol(true);
NACLib::TDiffACL acl;
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "1@" BUILTIN_ACL_DOMAIN);
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "2@" BUILTIN_ACL_DOMAIN);
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user1@" BUILTIN_ACL_DOMAIN);
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user2@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString());
+ server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString());
WaitACLModification();
auto ticket1 = "1@" BUILTIN_ACL_DOMAIN;
auto ticket2 = "2@" BUILTIN_ACL_DOMAIN;
- Cerr << "=== Writer - do reads\n";
- writer.Read(shortTopic2Name, "user1", ticket1, false, false, true);
+ Cerr << "=== Writer - do reads\n";
+ writer.Read(shortTopic2Name, "user1", ticket1, false, false, true);
- writer.Read(shortTopic2Name, "user1", "user2@" BUILTIN_ACL_DOMAIN, false, false, true);
- writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, true, false, true); //for topic
- writer.Read(shortTopic2Name, "user1", "user1@" BUILTIN_ACL_DOMAIN, true, false, true); //for consumer
- writer.Read(shortTopic2Name, "user2", ticket1, true, false, true);
- writer.Read(shortTopic2Name, "user2", ticket2, false, false, true);
+ writer.Read(shortTopic2Name, "user1", "user2@" BUILTIN_ACL_DOMAIN, false, false, true);
+ writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, true, false, true); //for topic
+ writer.Read(shortTopic2Name, "user1", "user1@" BUILTIN_ACL_DOMAIN, true, false, true); //for consumer
+ writer.Read(shortTopic2Name, "user2", ticket1, true, false, true);
+ writer.Read(shortTopic2Name, "user2", ticket2, false, false, true);
- writer.Read(shortTopic2Name, "user5", ticket1, true, false, true);
- writer.Read(shortTopic2Name, "user5", ticket2, true, false, true);
+ writer.Read(shortTopic2Name, "user5", ticket1, true, false, true);
+ writer.Read(shortTopic2Name, "user5", ticket2, true, false, true);
acl.Clear();
acl.AddAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, "user3@" BUILTIN_ACL_DOMAIN);
- server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString());
+ server.AnnoyingClient->ModifyACL("/Root/PQ", topic2, acl.SerializeAsString());
WaitACLModification();
- Cerr << "==== Writer - read\n";
- writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, false, true, true);
+ Cerr << "==== Writer - read\n";
+ writer.Read(shortTopic2Name, "user1", "user3@" BUILTIN_ACL_DOMAIN, false, true, true);
- auto Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ auto Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
auto StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_);
/* auto driver = server.AnnoyingClient->GetDriver();
- Cerr << "==== Start consuming loop\n";
+ Cerr << "==== Start consuming loop\n";
for (ui32 i = 0; i < 2; ++i) {
std::shared_ptr<NYdb::ICredentialsProviderFactory> creds = std::make_shared<TTestCredentialsProviderFactory>(NYdb::TStringType("user3@" BUILTIN_ACL_DOMAIN));
@@ -1217,7 +1217,7 @@ namespace {
ev->Confirm();
- Cerr << "=== Wait for consumer death (" << i << ")" << Endl;
+ Cerr << "=== Wait for consumer death (" << i << ")" << Endl;
msg = reader->GetEvent(true, 1);
UNIT_ASSERT(msg);
@@ -1229,15 +1229,15 @@ namespace {
UNIT_ASSERT(closeEv);
}
*/
- Cerr << "==== Start second loop\n";
+ Cerr << "==== Start second loop\n";
for (ui32 i = 0; i < 3; ++i){
- server.AnnoyingClient->GetClientInfo({topic2}, "user1", true);
+ server.AnnoyingClient->GetClientInfo({topic2}, "user1", true);
ReadInfoRequest request;
ReadInfoResponse response;
request.mutable_consumer()->set_path("user1");
request.set_get_only_original(true);
- request.add_topics()->set_path(shortTopic2Name);
+ request.add_topics()->set_path(shortTopic2Name);
grpc::ClientContext rcontext;
if (i == 0) {
rcontext.AddMetadata("x-ydb-auth-ticket", "user_without_rights@" BUILTIN_ACL_DOMAIN);
@@ -1259,11 +1259,11 @@ namespace {
}
Y_UNIT_TEST(CheckKillBalancer) {
- NPersQueue::TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY});
- PrepareForGrpc(server);
+ NPersQueue::TTestServer server;
+ server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY});
+ PrepareForGrpc(server);
- TPQDataWriter writer("source1", server);
+ TPQDataWriter writer("source1", server);
auto driver = server.AnnoyingClient->GetDriver();
@@ -1284,7 +1284,7 @@ namespace {
ev->Confirm();
}
- server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), "rt3.dc1--topic1");
+ server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), "rt3.dc1--topic1");
Cerr << "Balancer killed\n";
@@ -1316,20 +1316,20 @@ namespace {
Y_UNIT_TEST(TestWriteStat) {
- NPersQueue::TTestServer server(PQSettings(0, 1, true, "10"), false);
- auto netDataUpdated = server.PrepareNetDataFile(FormNetData());
- UNIT_ASSERT(netDataUpdated);
- server.StartServer();
+ NPersQueue::TTestServer server(PQSettings(0, 1, true, "10"), false);
+ auto netDataUpdated = server.PrepareNetDataFile(FormNetData());
+ UNIT_ASSERT(netDataUpdated);
+ server.StartServer();
- server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::NET_CLASSIFIER });
- server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_ERROR);
+ server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::NET_CLASSIFIER });
+ server.EnableLogs({ NKikimrServices::PERSQUEUE }, NActors::NLog::PRI_ERROR);
- auto sender = server.CleverServer->GetRuntime()->AllocateEdgeActor();
+ auto sender = server.CleverServer->GetRuntime()->AllocateEdgeActor();
- GetClassifierUpdate(*server.CleverServer, sender); //wait for initializing
+ GetClassifierUpdate(*server.CleverServer, sender); //wait for initializing
- server.AnnoyingClient->CreateTopic("rt3.dc1--account--topic1", 10, 10000, 10000, 2000);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10, 10000, 10000, 2000);
+ server.AnnoyingClient->CreateTopic("rt3.dc1--account--topic1", 10, 10000, 10000, 2000);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 10, 10000, 10000, 2000);
auto driver = server.AnnoyingClient->GetDriver();
@@ -1366,7 +1366,7 @@ namespace {
UNIT_ASSERT(ack->Acks.back().Stat->TotalTimeInPartitionQueue <= ack->Acks.back().Stat->PartitionQuotedTime + TDuration::Seconds(10));
}
}
- GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "writeSession", "account/topic1", "Vla");
+ GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "writeSession", "account/topic1", "Vla");
{
NYdb::NPersQueue::TReadSessionSettings settings;
settings.ConsumerName("shared/user").AppendTopics(TString("account/topic1")).ReadOriginal({"dc1"});
@@ -1378,14 +1378,14 @@ namespace {
Cerr << NYdb::NPersQueue::DebugString(*msg) << "\n";
}
- GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "readSession", "account/topic1", "Vla");
+ GetCounters(server.CleverServer->GetRuntime()->GetMonPort(), "pqproxy", "readSession", "account/topic1", "Vla");
}
Y_UNIT_TEST(TestWriteSessionsConflicts) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- TPQDataWriter writer("source", server);
+ TPQDataWriter writer("source", server);
server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
@@ -1433,13 +1433,13 @@ namespace {
Y_UNIT_TEST(TestLockErrors) {
return; // Test is ignored. FIX: KIKIMR-7881
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- auto pqLib = TPQLib::WithCerrLogger();
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ auto pqLib = TPQLib::WithCerrLogger();
{
- auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123");
+ auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123");
for (ui32 i = 1; i <= 11; ++i) {
auto f = producer->Write(i, TString(10, 'a'));
f.Wait();
@@ -1448,17 +1448,17 @@ namespace {
TConsumerSettings ss;
ss.Consumer = "user";
- ss.Server = TServerSetting{"localhost", server.GrpcPort};
- ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
+ ss.Server = TServerSetting{"localhost", server.GrpcPort};
+ ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
ss.MaxCount = 1;
ss.Unpack = false;
- auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
+ auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
auto msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0);
@@ -1470,12 +1470,12 @@ namespace {
future.Wait();
Cerr << future.GetValue() << "\n";
- std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
+ std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0);
@@ -1487,12 +1487,12 @@ namespace {
future.Wait();
Cerr << future.GetValue() << "\n";
- std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
+ std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0);
@@ -1504,12 +1504,12 @@ namespace {
future.Wait();
Cerr << future.GetValue() << "\n";
- std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
+ std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 0);
@@ -1530,12 +1530,12 @@ namespace {
}
}
- std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
+ std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 5);
@@ -1545,12 +1545,12 @@ namespace {
pp.SetValue(TAssignInfo{11, 11, false});
Sleep(TDuration::Seconds(5));
- std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
+ std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 11);
@@ -1562,12 +1562,12 @@ namespace {
future.Wait();
Cerr << future.GetValue() << "\n";
- std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
+ std::tie(consumer, ccResult) = CreateConsumer(pqLib, ss);
msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
UNIT_ASSERT(msg.GetValue().Response.assigned().read_offset() == 11);
@@ -1581,15 +1581,15 @@ namespace {
*/
Y_UNIT_TEST(TestBigMessage) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
server.EnableLogs({ NKikimrServices::PQ_WRITE_PROXY });
TPQDataWriter writer2("source", server);
auto driver = server.AnnoyingClient->GetDriver();
-
+
auto writer = CreateWriter(*driver, SHORT_TOPIC_NAME, "123", 0, "raw");
auto msg = writer->GetEvent(true);
@@ -1599,28 +1599,28 @@ namespace {
UNIT_ASSERT(ev);
writer->Write(std::move(ev->ContinuationToken), TString((1 << 20) * 60, 'a')); //TODO: Increase GRPC_ARG_MAX_SEND_MESSAGE_LENGTH
- {
- msg = writer->GetEvent(true);
- UNIT_ASSERT(msg); // ReadyToAcceptEvent
- auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TReadyToAcceptEvent>(&*msg);
- UNIT_ASSERT(ev2);
- }
- {
- msg = writer->GetEvent(true);
- UNIT_ASSERT(msg); // Ack on write
- auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TAcksEvent>(&*msg);
- UNIT_ASSERT(ev2);
- }
+ {
+ msg = writer->GetEvent(true);
+ UNIT_ASSERT(msg); // ReadyToAcceptEvent
+ auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TReadyToAcceptEvent>(&*msg);
+ UNIT_ASSERT(ev2);
+ }
+ {
+ msg = writer->GetEvent(true);
+ UNIT_ASSERT(msg); // Ack on write
+ auto ev2 = std::get_if<NYdb::NPersQueue::TWriteSessionEvent::TAcksEvent>(&*msg);
+ UNIT_ASSERT(ev2);
+ }
}
/*
void TestRereadsWhenDataIsEmptyImpl(bool withWait) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- TPQDataWriter writer("source", server);
- auto pqLib = TPQLib::WithCerrLogger();
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ TPQDataWriter writer("source", server);
+ auto pqLib = TPQLib::WithCerrLogger();
// Write nonempty data
NKikimr::NPersQueueTests::TRequestWritePQ writeReq(DEFAULT_TOPIC_NAME, 0, "src", 4);
@@ -1635,7 +1635,7 @@ namespace {
}
TString serialized;
UNIT_ASSERT(dataChunk.SerializeToString(&serialized));
- server.AnnoyingClient->WriteToPQ(writeReq, serialized);
+ server.AnnoyingClient->WriteToPQ(writeReq, serialized);
};
write("data1");
write("data2", true);
@@ -1647,10 +1647,10 @@ namespace {
bool unpack = false;
ui32 maxInflyRequests = 1;
ui32 maxMemoryUsage = 1;
- auto [consumer, ccResult] = CreateConsumer(
- pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}},
- maxCount, unpack, {}, maxInflyRequests, maxMemoryUsage
- );
+ auto [consumer, ccResult] = CreateConsumer(
+ pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}},
+ maxCount, unpack, {}, maxInflyRequests, maxMemoryUsage
+ );
UNIT_ASSERT_C(ccResult.Response.response_case() == MigrationStreamingReadServerMessage::kInitResponse, ccResult.Response);
auto msg1 = GetNextMessageSkipAssignment(consumer).GetValueSync().Response;
@@ -1695,32 +1695,32 @@ namespace {
Y_UNIT_TEST(TestLockAfterDrop) {
- NPersQueue::TTestServer server{false};
- server.GrpcServerOptions.SetMaxMessageSize(130*1024*1024);
- server.StartServer();
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.WaitInit(SHORT_TOPIC_NAME);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- auto pqLib = TPQLib::WithCerrLogger();
-
- auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123");
+ NPersQueue::TTestServer server{false};
+ server.GrpcServerOptions.SetMaxMessageSize(130*1024*1024);
+ server.StartServer();
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ server.WaitInit(SHORT_TOPIC_NAME);
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ auto pqLib = TPQLib::WithCerrLogger();
+
+ auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, "123");
auto f = producer->Write(1, TString(1024, 'a'));
f.Wait();
ui32 maxCount = 1;
bool unpack = false;
- auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack);
+ auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack);
Cerr << ccResult.Response << "\n";
auto msg = consumer->GetNextMessage();
msg.Wait();
UNIT_ASSERT_C(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned, msg.GetValue().Response);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.assigned().partition() == 0);
- server.CleverServer->GetRuntime()->ResetScheduledCount();
- server.AnnoyingClient->RestartPartitionTablets(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
+ server.CleverServer->GetRuntime()->ResetScheduledCount();
+ server.AnnoyingClient->RestartPartitionTablets(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
msg.GetValue().StartRead.SetValue({0,0,false});
@@ -1733,16 +1733,16 @@ namespace {
Y_UNIT_TEST(TestMaxNewTopicModel) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"__extra_path_symbols_allowed", "@"}});
- server.AnnoyingClient->CreateTopic("rt3.dc1--aaa@bbb@ccc--topic", 1);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->AlterUserAttributes("/", "Root", {{"__extra_path_symbols_allowed", "@"}});
+ server.AnnoyingClient->CreateTopic("rt3.dc1--aaa@bbb@ccc--topic", 1);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- auto pqLib = TPQLib::WithCerrLogger();
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ auto pqLib = TPQLib::WithCerrLogger();
{
- auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, "aaa/bbb/ccc/topic", "123");
+ auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, "aaa/bbb/ccc/topic", "123");
UNIT_ASSERT_C(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse, pcResult.Response);
for (ui32 i = 1; i <= 11; ++i) {
auto f = producer->Write(i, TString(10, 'a'));
@@ -1753,7 +1753,7 @@ namespace {
ui32 maxCount = 1;
bool unpack = false;
- auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {"aaa/bbb/ccc/topic", {}}, maxCount, unpack);
+ auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {"aaa/bbb/ccc/topic", {}}, maxCount, unpack);
UNIT_ASSERT_C(ccResult.Response.response_case() == MigrationStreamingReadServerMessage::kInitResponse, ccResult.Response);
auto msg = consumer->GetNextMessage();
@@ -1764,18 +1764,18 @@ namespace {
Y_UNIT_TEST(TestReleaseWithAssigns) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- auto pqLib = TPQLib::WithCerrLogger();
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ auto pqLib = TPQLib::WithCerrLogger();
- TPQDataWriter writer("source", server);
+ TPQDataWriter writer("source", server);
for (ui32 i = 1; i <= 3; ++i) {
TString sourceId = "123" + ToString<int>(i);
ui32 partitionGroup = i;
- auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup);
+ auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup);
UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse);
auto f = producer->Write(i, TString(10, 'a'));
@@ -1784,13 +1784,13 @@ namespace {
TConsumerSettings ss;
ss.Consumer = "user";
- ss.Server = TServerSetting{"localhost", server.GrpcPort};
- ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
+ ss.Server = TServerSetting{"localhost", server.GrpcPort};
+ ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
ss.ReadMirroredPartitions = false;
ss.MaxCount = 3;
ss.Unpack = false;
- auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
+ auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
Cerr << ccResult.Response << "\n";
for (ui32 i = 1; i <= 3; ++i) {
@@ -1798,11 +1798,11 @@ namespace {
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
}
- auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss);
+ auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss);
Cerr << ccResult2.Response << "\n";
auto msg = consumer->GetNextMessage();
@@ -1811,7 +1811,7 @@ namespace {
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kRelease);
- UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.release().cluster() == "dc1");
UNIT_ASSERT(!msg2.Wait(TDuration::Seconds(1)));
@@ -1822,18 +1822,18 @@ namespace {
Cerr << msg2.GetValue().Response << "\n";
UNIT_ASSERT(msg2.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg2.GetValue().Response.assigned().cluster() == "dc1");
}
Y_UNIT_TEST(TestSilentRelease) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 3);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- auto pqLib = TPQLib::WithCerrLogger();
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ auto pqLib = TPQLib::WithCerrLogger();
- TPQDataWriter writer("source", server);
+ TPQDataWriter writer("source", server);
TVector<std::pair<ui64, ui64>> cookies;
@@ -1841,8 +1841,8 @@ namespace {
TString sourceId = "123" + ToString<int>(i);
ui32 partitionGroup = i;
- auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup);
- Cerr << "===Response: " << pcResult.Response << Endl;
+ auto [producer, pcResult] = CreateProducer(pqLib, server.GrpcPort, SHORT_TOPIC_NAME, sourceId, partitionGroup);
+ Cerr << "===Response: " << pcResult.Response << Endl;
UNIT_ASSERT(pcResult.Response.server_message_case() == StreamingWriteServerMessage::kInitResponse);
auto f = producer->Write(i, TString(10, 'a'));
f.Wait();
@@ -1850,13 +1850,13 @@ namespace {
TConsumerSettings ss;
ss.Consumer = "user";
- ss.Server = TServerSetting{"localhost", server.GrpcPort};
- ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
+ ss.Server = TServerSetting{"localhost", server.GrpcPort};
+ ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
ss.ReadMirroredPartitions = false;
ss.MaxCount = 1;
ss.Unpack = false;
- auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
+ auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
Cerr << ccResult.Response << "\n";
for (ui32 i = 1; i <= 3; ++i) {
@@ -1868,7 +1868,7 @@ namespace {
}
}
- auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss);
+ auto [consumer2, ccResult2] = CreateConsumer(pqLib, ss);
Cerr << ccResult2.Response << "\n";
auto msg = consumer->GetNextMessage();
@@ -1884,7 +1884,7 @@ namespace {
Cerr << msg2.GetValue().Response << "\n";
UNIT_ASSERT(msg2.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg2.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg2.GetValue().Response.assigned().cluster() == "dc1");
UNIT_ASSERT(msg2.GetValue().Response.assigned().read_offset() == 1);
}
@@ -1893,30 +1893,30 @@ namespace {
/*
Y_UNIT_TEST(TestDoubleRelease) {
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(
DEFAULT_TOPIC_NAME, 1, 8000000, 86400, 50000000, "", 50000000, {"user1"}, {"user1", "user2"}
- );
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::TABLET_AGGREGATOR });
- auto pqLib = TPQLib::WithCerrLogger();
- TPQDataWriter writer("source", server);
+ );
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::TABLET_AGGREGATOR });
+ auto pqLib = TPQLib::WithCerrLogger();
+ TPQDataWriter writer("source", server);
TConsumerSettings ss;
ss.Consumer = "user";
- ss.Server = TServerSetting{"localhost", server.GrpcPort};
- ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
+ ss.Server = TServerSetting{"localhost", server.GrpcPort};
+ ss.Topics.push_back({SHORT_TOPIC_NAME, {}});
ss.ReadMirroredPartitions = false;
ss.MaxCount = 3;
ss.Unpack = false;
- auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
+ auto [consumer, ccResult] = CreateConsumer(pqLib, ss);
Cerr << ccResult.Response << "\n";
auto msg = consumer->GetNextMessage();
msg.Wait();
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kAssigned);
- UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.assigned().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.assigned().cluster() == "dc1");
msg = consumer->GetNextMessage();
@@ -1924,36 +1924,36 @@ namespace {
THolder<IConsumer> consumer2;
do {
- std::tie(consumer2, ccResult) = CreateConsumer(pqLib, ss);
+ std::tie(consumer2, ccResult) = CreateConsumer(pqLib, ss);
Cerr << ccResult.Response << "\n";
} while(!msg.Wait(TDuration::Seconds(1)));
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kRelease);
- UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.release().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.release().forceful_release() == false);
msg = consumer->GetNextMessage();
UNIT_ASSERT(!msg.Wait(TDuration::Seconds(1)));
- server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
+ server.AnnoyingClient->RestartBalancerTablet(server.CleverServer->GetRuntime(), DEFAULT_TOPIC_NAME);
UNIT_ASSERT(msg.Wait(TDuration::Seconds(1)));
Cerr << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Response.response_case() == MigrationStreamingReadServerMessage::kRelease);
- UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME);
+ UNIT_ASSERT(msg.GetValue().Response.release().topic().path() == SHORT_TOPIC_NAME);
UNIT_ASSERT(msg.GetValue().Response.release().cluster() == "dc1");
UNIT_ASSERT(msg.GetValue().Response.release().forceful_release() == true);
THolder<TEvTabletCounters::TEvTabletLabeledCountersResponse> response;
- TActorId edge = server.CleverServer->GetRuntime()->AllocateEdgeActor();
+ TActorId edge = server.CleverServer->GetRuntime()->AllocateEdgeActor();
do {
auto actorId = NKikimr::CreateClusterLabeledCountersAggregatorActor(edge, TTabletTypes::PersQueue, 3, "rt3.*--*,user*!!/!!*!!/rt3.*--*", 0); // remove !!
- server.CleverServer->GetRuntime()->Register(actorId);
- response = server.CleverServer->GetRuntime()->
+ server.CleverServer->GetRuntime()->Register(actorId);
+ response = server.CleverServer->GetRuntime()->
GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>();
Cerr << "FINAL RESPONSE :\n" << response->Record.DebugString() << Endl;
@@ -1962,8 +1962,8 @@ namespace {
Cerr << "MULITREQUEST\n";
auto actorId = NKikimr::CreateClusterLabeledCountersAggregatorActor(edge, TTabletTypes::PersQueue, 3, "rt3.*--*,user*!!/!!*!!/rt3.*--*", 3); // remove !!
- server.CleverServer->GetRuntime()->Register(actorId);
- response = server.CleverServer->GetRuntime()->
+ server.CleverServer->GetRuntime()->Register(actorId);
+ response = server.CleverServer->GetRuntime()->
GrabEdgeEvent<TEvTabletCounters::TEvTabletLabeledCountersResponse>();
Cerr << "FINAL RESPONSE2 :\n" << response->Record.DebugString() << Endl;
@@ -1986,9 +1986,9 @@ namespace {
const auto& writeResponse = writeResult.GetValueSync().Response;
UNIT_ASSERT_EQUAL_C(Ydb::StatusIds::SUCCESS, writeResponse.status(), "Response: " << writeResponse);
- auto pqLib = TPQLib::WithCerrLogger();
+ auto pqLib = TPQLib::WithCerrLogger();
using namespace NPersQueue;
- auto [consumer, ccResult] = CreateConsumer(pqLib, setup.GetGrpcPort(), "shared/user", {"/account/topic" , {}}, 1000, false);
+ auto [consumer, ccResult] = CreateConsumer(pqLib, setup.GetGrpcPort(), "shared/user", {"/account/topic" , {}}, 1000, false);
Cerr << ccResult.Response << "\n";
auto msg = consumer->GetNextMessage();
@@ -2025,17 +2025,17 @@ namespace {
THolder<IProducer> producer = setup.StartProducer(topicPath, true);
- auto pqLib = TPQLib::WithCerrLogger();
-
- auto [consumer, ccResult] = CreateConsumer(
- pqLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false
- );
+ auto pqLib = TPQLib::WithCerrLogger();
+
+ auto [consumer, ccResult] = CreateConsumer(
+ pqLib, setup.GetGrpcPort(), consumerPath, {topicPath , {}}, 1000, false
+ );
Cerr << ccResult.Response << "\n";
{
auto msg = consumer->GetNextMessage();
msg.Wait();
- Cerr << "consumer assign response: " << msg.GetValue().Response << "\n";
+ Cerr << "consumer assign response: " << msg.GetValue().Response << "\n";
UNIT_ASSERT(msg.GetValue().Type == EMT_ASSIGNED);
msg.GetValue().StartRead.SetValue(TAssignInfo());
}
@@ -2110,23 +2110,23 @@ namespace {
return;
}
- NPersQueue::TTestServer server;
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.WaitInit(SHORT_TOPIC_NAME);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
+ NPersQueue::TTestServer server;
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ server.WaitInit(SHORT_TOPIC_NAME);
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY });
- server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::OK, false);
- auto pqLib = TPQLib::WithCerrLogger();
+ server.AnnoyingClient->DeleteTopic2(DEFAULT_TOPIC_NAME, NPersQueue::NErrorCode::OK, false);
+ auto pqLib = TPQLib::WithCerrLogger();
ui32 maxCount = 1;
bool unpack = false;
- auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack);
- Cerr << "Consumer create response: " << ccResult.Response << "\n";
+ auto [consumer, ccResult] = CreateConsumer(pqLib, server.GrpcPort, "user", {SHORT_TOPIC_NAME, {}}, maxCount, unpack);
+ Cerr << "Consumer create response: " << ccResult.Response << "\n";
auto isDead = consumer->IsDead();
isDead.Wait();
- Cerr << "Is dead future value: " << isDead.GetValue() << "\n";
- UNIT_ASSERT_EQUAL(ccResult.Response.Getissues(0).issue_code(), Ydb::PersQueue::ErrorCode::UNKNOWN_TOPIC);
+ Cerr << "Is dead future value: " << isDead.GetValue() << "\n";
+ UNIT_ASSERT_EQUAL(ccResult.Response.Getissues(0).issue_code(), Ydb::PersQueue::ErrorCode::UNKNOWN_TOPIC);
}
*/
Y_UNIT_TEST(Codecs_InitWriteSession_DefaultTopicSupportedCodecsInInitResponse) {
@@ -2352,26 +2352,26 @@ namespace {
}
Y_UNIT_TEST(SchemeOperationsTest) {
- NPersQueue::TTestServer server;
- server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
- TString topic1 = "rt3.dc1--acc--topic1";
- TString topic3 = "rt3.dc1--acc--topic3";
- server.AnnoyingClient->CreateTopic(topic1, 1);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.AnnoyingClient->CreateConsumer("user");
+ NPersQueue::TTestServer server;
+ server.EnableLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
+ TString topic1 = "rt3.dc1--acc--topic1";
+ TString topic3 = "rt3.dc1--acc--topic3";
+ server.AnnoyingClient->CreateTopic(topic1, 1);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ server.AnnoyingClient->CreateConsumer("user");
std::shared_ptr<grpc::Channel> Channel_;
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_;
{
- Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_);
}
do {
CreateTopicRequest request;
CreateTopicResponse response;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
auto props = request.mutable_settings();
props->set_partitions_count(1);
props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
@@ -2401,7 +2401,7 @@ namespace {
// local cluster
CreateTopicRequest request;
CreateTopicResponse response;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
auto props = request.mutable_settings();
props->set_partitions_count(2);
props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
@@ -2419,7 +2419,7 @@ namespace {
response.operation().result().UnpackTo(&res);
Cerr << response << "\n" << res << "\n";
UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
- server.AnnoyingClient->AddTopic(topic3);
+ server.AnnoyingClient->AddTopic(topic3);
}
auto alter =[&StubP_](const AlterTopicRequest& request, Ydb::StatusIds::StatusCode statusCode, bool auth)
@@ -2440,7 +2440,7 @@ namespace {
};
AlterTopicRequest request;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
auto props = request.mutable_settings();
props->set_partitions_count(1);
props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE);
@@ -2457,7 +2457,7 @@ namespace {
props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1));
auto rr = props->add_read_rules();
alter(request, Ydb::StatusIds::BAD_REQUEST, false);
- server.AnnoyingClient->AlterTopic();
+ server.AnnoyingClient->AlterTopic();
props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW);
props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD);
@@ -2481,9 +2481,9 @@ namespace {
(*props->mutable_attributes())["_partitions_per_tablet"] = "5";
alter(request, Ydb::StatusIds::SUCCESS, false);
- TString topic4 = "rt3.dc1--acc--topic4";
- server.AnnoyingClient->CreateTopic(topic4, 1); //ensure creation
- auto res = server.AnnoyingClient->DescribeTopic({topic3});
+ TString topic4 = "rt3.dc1--acc--topic4";
+ server.AnnoyingClient->CreateTopic(topic4, 1); //ensure creation
+ auto res = server.AnnoyingClient->DescribeTopic({topic3});
Cerr << res.DebugString();
TString resultDescribe = R"___(TopicInfo {
Topic: "rt3.dc1--acc--topic3"
@@ -2572,7 +2572,7 @@ namespace {
{
DescribeTopicRequest request;
DescribeTopicResponse response;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
grpc::ClientContext rcontext;
rcontext.AddMetadata("x-ydb-auth-ticket", "user@" BUILTIN_ACL_DOMAIN);
@@ -2603,7 +2603,7 @@ namespace {
{
DescribeTopicRequest request;
DescribeTopicResponse response;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
grpc::ClientContext rcontext;
auto status = StubP_->DescribeTopic(&rcontext, request, &response);
@@ -2623,7 +2623,7 @@ namespace {
{
DescribeTopicRequest request;
DescribeTopicResponse response;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
grpc::ClientContext rcontext;
auto status = StubP_->DescribeTopic(&rcontext, request, &response);
@@ -2639,7 +2639,7 @@ namespace {
{
DropTopicRequest request;
DropTopicResponse response;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
grpc::ClientContext rcontext;
auto status = StubP_->DropTopic(&rcontext, request, &response);
@@ -2648,15 +2648,15 @@ namespace {
response.operation().result().UnpackTo(&res);
Cerr << response << "\n" << res << "\n";
UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS);
- server.AnnoyingClient->RemoveTopic(topic3);
+ server.AnnoyingClient->RemoveTopic(topic3);
}
{
DropTopicRequest request;
DropTopicResponse response;
- request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
-
+ request.set_path(TStringBuilder() << "/Root/PQ/" << topic3);
+
grpc::ClientContext rcontext;
auto status = StubP_->DropTopic(&rcontext, request, &response);
@@ -2667,13 +2667,13 @@ namespace {
UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SCHEME_ERROR);
}
- server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic5", 1); //ensure creation
- server.AnnoyingClient->DescribeTopic({topic3}, true);
+ server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic5", 1); //ensure creation
+ server.AnnoyingClient->DescribeTopic({topic3}, true);
{
NYdb::TDriverConfig driverCfg;
- driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort);
+ driverCfg.SetEndpoint(TStringBuilder() << "localhost:" << server.GrpcPort);
std::shared_ptr<NYdb::TDriver> ydbDriver(new NYdb::TDriver(driverCfg));
auto pqClient = NYdb::NPersQueue::TPersQueueClient(*ydbDriver);
@@ -2686,18 +2686,18 @@ namespace {
Y_UNIT_TEST(SchemeOperationsCheckPropValues) {
- NPersQueue::TTestServer server;
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
+ NPersQueue::TTestServer server;
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
- server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1);
- server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
- server.AnnoyingClient->CreateConsumer("user");
+ server.AnnoyingClient->CreateTopic("rt3.dc1--acc--topic1", 1);
+ server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 1);
+ server.AnnoyingClient->CreateConsumer("user");
std::shared_ptr<grpc::Channel> Channel_;
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> StubP_;
{
- Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ Channel_ = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
StubP_ = Ydb::PersQueue::V1::PersQueueService::NewStub(Channel_);
}
@@ -2764,19 +2764,19 @@ namespace {
}
Y_UNIT_TEST(ReadRuleServiceType) {
- TServerSettings settings = PQSettings(0);
+ TServerSettings settings = PQSettings(0);
{
settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
settings.PQConfig.AddClientServiceType()->SetName("SecondType");
}
- NPersQueue::TTestServer server(settings);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
+ NPersQueue::TTestServer server(settings);
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
{
- std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
}
auto checkDescribe = [&](const TVector<std::pair<TString, TString>>& readRules) {
@@ -3166,16 +3166,16 @@ namespace {
}
Y_UNIT_TEST(ReadRuleServiceTypeMigration) {
- TServerSettings settings = PQSettings(0);
+ TServerSettings settings = PQSettings(0);
{
settings.PQConfig.MutableDefaultClientServiceType()->SetName("default_type");
settings.PQConfig.AddClientServiceType()->SetName("MyGreatType");
settings.PQConfig.AddClientServiceType()->SetName("AnotherType");
settings.PQConfig.AddClientServiceType()->SetName("SecondType");
}
- NPersQueue::TTestServer server(settings);
+ NPersQueue::TTestServer server(settings);
- server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
+ server.EnableLogs({ NKikimrServices::PQ_READ_PROXY, NKikimrServices::BLACKBOX_VALIDATOR });
const ui32 topicsCount = 4;
for (ui32 i = 1; i <= topicsCount; ++i) {
@@ -3188,7 +3188,7 @@ namespace {
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> pqStub;
{
- std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
+ std::shared_ptr<grpc::Channel> channel = grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials());
pqStub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel);
}
auto doAlter = [&](const TString& topic, const TVector<std::pair<TString, TString>>& readRules) {
diff --git a/ydb/services/persqueue_v1/persqueue_utils.cpp b/ydb/services/persqueue_v1/persqueue_utils.cpp
index 7d21e2314a..51d7f38045 100644
--- a/ydb/services/persqueue_v1/persqueue_utils.cpp
+++ b/ydb/services/persqueue_v1/persqueue_utils.cpp
@@ -2,85 +2,85 @@
#include <ydb/core/base/path.h>
-namespace NKikimr::NGRpcProxy::V1 {
+namespace NKikimr::NGRpcProxy::V1 {
-TAclWrapper::TAclWrapper(THolder<NACLib::TSecurityObject> acl)
- : AclOldSchemeCache(std::move(acl))
-{
- Y_VERIFY(AclOldSchemeCache);
-}
+TAclWrapper::TAclWrapper(THolder<NACLib::TSecurityObject> acl)
+ : AclOldSchemeCache(std::move(acl))
+{
+ Y_VERIFY(AclOldSchemeCache);
+}
+
+TAclWrapper::TAclWrapper(TIntrusivePtr<TSecurityObject> acl)
+ : AclNewSchemeCache(std::move(acl))
+{
+ Y_VERIFY(AclNewSchemeCache);
+}
-TAclWrapper::TAclWrapper(TIntrusivePtr<TSecurityObject> acl)
- : AclNewSchemeCache(std::move(acl))
-{
- Y_VERIFY(AclNewSchemeCache);
-}
-
-bool TAclWrapper::CheckAccess(NACLib::EAccessRights rights, const NACLib::TUserToken& userToken) {
- if (AclOldSchemeCache) {
- return AclOldSchemeCache->CheckAccess(rights, userToken);
- } else {
- return AclNewSchemeCache->CheckAccess(rights, userToken);
+bool TAclWrapper::CheckAccess(NACLib::EAccessRights rights, const NACLib::TUserToken& userToken) {
+ if (AclOldSchemeCache) {
+ return AclOldSchemeCache->CheckAccess(rights, userToken);
+ } else {
+ return AclNewSchemeCache->CheckAccess(rights, userToken);
}
-}
+}
+
+using namespace NSchemeCache;
-using namespace NSchemeCache;
-
-TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEntry& entry) {
- auto fullPath = JoinPath(entry.Path);
- auto& topicName = entry.Path.back();
- switch (entry.Status) {
- case TSchemeCacheNavigate::EStatus::RootUnknown : {
- return TProcessingResult {
- Ydb::PersQueue::ErrorCode::ErrorCode::BAD_REQUEST,
- Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14",
- fullPath.c_str(), entry.Path[0].c_str()),
- true
- };
+TProcessingResult ProcessMetaCacheTopicResponse(const TSchemeCacheNavigate::TEntry& entry) {
+ auto fullPath = JoinPath(entry.Path);
+ auto& topicName = entry.Path.back();
+ switch (entry.Status) {
+ case TSchemeCacheNavigate::EStatus::RootUnknown : {
+ return TProcessingResult {
+ Ydb::PersQueue::ErrorCode::ErrorCode::BAD_REQUEST,
+ Sprintf("path '%s' has unknown/invalid root prefix '%s', Marker# PQ14",
+ fullPath.c_str(), entry.Path[0].c_str()),
+ true
+ };
+ }
+ case TSchemeCacheNavigate::EStatus::PathErrorUnknown: {
+ return TProcessingResult {
+ Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC,
+ Sprintf("no path '%s', Marker# PQ15", fullPath.c_str()),
+ true
+ };
+ }
+ case TSchemeCacheNavigate::EStatus::Ok:
+ break;
+ default: {
+ return TProcessingResult {
+ Ydb::PersQueue::ErrorCode::ErrorCode::ERROR,
+ Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1",
+ topicName.c_str(), ToString(entry.Status).c_str()),
+ true
+ };
}
- case TSchemeCacheNavigate::EStatus::PathErrorUnknown: {
- return TProcessingResult {
- Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC,
- Sprintf("no path '%s', Marker# PQ15", fullPath.c_str()),
- true
- };
- }
- case TSchemeCacheNavigate::EStatus::Ok:
- break;
- default: {
- return TProcessingResult {
- Ydb::PersQueue::ErrorCode::ErrorCode::ERROR,
- Sprintf("topic '%s' describe error, Status# %s, Marker# PQ1",
- topicName.c_str(), ToString(entry.Status).c_str()),
- true
- };
- }
}
- if (entry.Kind != TSchemeCacheNavigate::KindTopic) {
- return TProcessingResult {
- Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC,
- Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()),
- true
- };
- }
- if (!entry.PQGroupInfo) {
- return TProcessingResult {
- Ydb::PersQueue::ErrorCode::ErrorCode::ERROR,
- Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ99",
- topicName.c_str()),
- true
- };
- }
- auto& description = entry.PQGroupInfo->Description;
- if (!description.HasBalancerTabletID() || description.GetBalancerTabletID() == 0) {
- return TProcessingResult {
- Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC,
- Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()),
- true
- };
- }
- return {};
+ if (entry.Kind != TSchemeCacheNavigate::KindTopic) {
+ return TProcessingResult {
+ Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC,
+ Sprintf("item '%s' is not a topic, Marker# PQ13", fullPath.c_str()),
+ true
+ };
+ }
+ if (!entry.PQGroupInfo) {
+ return TProcessingResult {
+ Ydb::PersQueue::ErrorCode::ErrorCode::ERROR,
+ Sprintf("topic '%s' describe error, reason: could not retrieve topic description, Marker# PQ99",
+ topicName.c_str()),
+ true
+ };
+ }
+ auto& description = entry.PQGroupInfo->Description;
+ if (!description.HasBalancerTabletID() || description.GetBalancerTabletID() == 0) {
+ return TProcessingResult {
+ Ydb::PersQueue::ErrorCode::ErrorCode::UNKNOWN_TOPIC,
+ Sprintf("topic '%s' has no balancer, Marker# PQ193", topicName.c_str()),
+ true
+ };
+ }
+ return {};
}
-
-} // namespace NKikimr::NGRpcProxy::V1
+
+} // namespace NKikimr::NGRpcProxy::V1
diff --git a/ydb/services/persqueue_v1/persqueue_utils.h b/ydb/services/persqueue_v1/persqueue_utils.h
index 188e47a1af..7860067982 100644
--- a/ydb/services/persqueue_v1/persqueue_utils.h
+++ b/ydb/services/persqueue_v1/persqueue_utils.h
@@ -1,32 +1,32 @@
#pragma once
#include <ydb/public/api/protos/persqueue_error_codes_v1.pb.h>
-
+
#include <ydb/library/aclib/aclib.h>
#include <ydb/core/scheme/scheme_tabledefs.h>
#include <ydb/core/base/counters.h>
#include <ydb/core/tx/scheme_cache/scheme_cache.h>
-namespace NKikimr::NGRpcProxy::V1 {
-
-class TAclWrapper {
-public:
- TAclWrapper(THolder<NACLib::TSecurityObject>);
- TAclWrapper(TIntrusivePtr<TSecurityObject>);
-
- bool CheckAccess(NACLib::EAccessRights, const NACLib::TUserToken& userToken);
-
-private:
- THolder<NACLib::TSecurityObject> AclOldSchemeCache;
- TIntrusivePtr<TSecurityObject> AclNewSchemeCache;
-};
-
-struct TProcessingResult {
- Ydb::PersQueue::ErrorCode::ErrorCode ErrorCode;
- TString Reason;
- bool IsFatal = false;
-};
-
-TProcessingResult ProcessMetaCacheTopicResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry);
-
-} //namespace NKikimr::NGRpcProxy::V1
+namespace NKikimr::NGRpcProxy::V1 {
+
+class TAclWrapper {
+public:
+ TAclWrapper(THolder<NACLib::TSecurityObject>);
+ TAclWrapper(TIntrusivePtr<TSecurityObject>);
+
+ bool CheckAccess(NACLib::EAccessRights, const NACLib::TUserToken& userToken);
+
+private:
+ THolder<NACLib::TSecurityObject> AclOldSchemeCache;
+ TIntrusivePtr<TSecurityObject> AclNewSchemeCache;
+};
+
+struct TProcessingResult {
+ Ydb::PersQueue::ErrorCode::ErrorCode ErrorCode;
+ TString Reason;
+ bool IsFatal = false;
+};
+
+TProcessingResult ProcessMetaCacheTopicResponse(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry);
+
+} //namespace NKikimr::NGRpcProxy::V1
diff --git a/ydb/services/persqueue_v1/ut/api_test_setup.h b/ydb/services/persqueue_v1/ut/api_test_setup.h
index 1e904b8ff9..8d5d82b33d 100644
--- a/ydb/services/persqueue_v1/ut/api_test_setup.h
+++ b/ydb/services/persqueue_v1/ut/api_test_setup.h
@@ -16,23 +16,23 @@ private:
using TStreamingWriteClientMessage = Ydb::PersQueue::V1::StreamingWriteClientMessage;
using TStreamingWriteServerMessage = Ydb::PersQueue::V1::StreamingWriteServerMessage;
TLog Log;
- NPersQueue::TTestServer server;
+ NPersQueue::TTestServer server;
std::shared_ptr<::grpc::Channel> channel;
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> service;
public:
APITestSetup(const TString& testCaseName)
: Log("cerr")
- , server(NPersQueue::TTestServer())
- , channel(grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()))
+ , server(NPersQueue::TTestServer())
+ , channel(grpc::CreateChannel("localhost:" + ToString(server.GrpcPort), grpc::InsecureChannelCredentials()))
, service(Ydb::PersQueue::V1::PersQueueService::NewStub(channel))
{
Log.SetFormatter([testCaseName](ELogPriority priority, TStringBuf message) {
return TStringBuilder() << TInstant::Now() << " :" << testCaseName << " " << priority << ": " << message << Endl;
});
- server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_DEBUG);
- server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NActors::NLog::PRI_DEBUG);
- server.AnnoyingClient->CreateTopic("rt3.dc1--" + TString(GetTestTopic()), 1);
- NKikimr::NPersQueueTests::TPQDataWriter writer(GetTestMessageGroupId(), server);
+ server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_WRITE_PROXY, NActors::NLog::PRI_DEBUG);
+ server.CleverServer->GetRuntime()->SetLogPriority(NKikimrServices::PQ_READ_PROXY, NActors::NLog::PRI_DEBUG);
+ server.AnnoyingClient->CreateTopic("rt3.dc1--" + TString(GetTestTopic()), 1);
+ NKikimr::NPersQueueTests::TPQDataWriter writer(GetTestMessageGroupId(), server);
auto seed = TInstant::Now().MicroSeconds();
// This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable
@@ -65,7 +65,7 @@ public:
}
NKikimr::Tests::TServer& GetServer() {
- return *server.CleverServer;
+ return *server.CleverServer;
}
TString GetPQRoot() {
@@ -73,11 +73,11 @@ public:
}
NKikimrPQ::TPQConfig& GetPQConfig() {
- return server.CleverServer->GetRuntime()->GetAppData().PQConfig;
+ return server.CleverServer->GetRuntime()->GetAppData().PQConfig;
}
NKikimr::NPersQueueTests::TFlatMsgBusPQClient& GetFlatMsgBusPQClient() {
- return *server.AnnoyingClient;
+ return *server.AnnoyingClient;
}
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub>& GetPersQueueService() {
@@ -112,7 +112,7 @@ public:
TServerMessage serverMessage;
Log << TLOG_INFO << "Wait for \"init_response\"";
AssertSuccessfullStreamingOperation(stream->Read(&serverMessage), stream);
- Cerr << "Init response: " << serverMessage.ShortDebugString() << Endl;
+ Cerr << "Init response: " << serverMessage.ShortDebugString() << Endl;
UNIT_ASSERT_C(serverMessage.server_message_case() == TServerMessage::kInitResponse, serverMessage);
Log << TLOG_INFO << "Session ID is " << serverMessage.init_response().session_id().Quote();
return serverMessage;
diff --git a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h
index 37432a7f7a..2933d1f3a1 100644
--- a/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h
+++ b/ydb/services/persqueue_v1/ut/persqueue_test_fixture.h
@@ -16,15 +16,15 @@
namespace NKikimr::NPersQueueTests {
-
-
-#define SET_LOCALS \
- auto& pqClient = server.Server->AnnoyingClient; \
- Y_UNUSED(pqClient); \
- auto* runtime = server.Server->CleverServer->GetRuntime(); \
- Y_UNUSED(runtime); \
-
-
+
+
+#define SET_LOCALS \
+ auto& pqClient = server.Server->AnnoyingClient; \
+ Y_UNUSED(pqClient); \
+ auto* runtime = server.Server->CleverServer->GetRuntime(); \
+ Y_UNUSED(runtime); \
+
+
using namespace Tests;
using namespace NKikimrClient;
using namespace Ydb::PersQueue;
@@ -32,46 +32,46 @@ namespace NKikimr::NPersQueueTests {
using namespace NThreading;
using namespace NNetClassifier;
- class TPersQueueV1TestServerBase {
+ class TPersQueueV1TestServerBase {
public:
- virtual void AlterSettings(NKikimr::Tests::TServerSettings& settings) {
- Y_UNUSED(settings);
- }
- void InitializePQ() {
- Cerr << "===Init PQ - create server\n";
- Y_VERIFY(Server == nullptr);
- PortManager = new TPortManager();
- Server = MakeHolder<NPersQueue::TTestServer>(false, PortManager);
- Server->ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(TenantModeEnabled());
- Server->ServerSettings.PQConfig.SetMetaCacheTimeoutSec(30);
- AlterSettings(Server->ServerSettings);
- Server->StartServer(false);
+ virtual void AlterSettings(NKikimr::Tests::TServerSettings& settings) {
+ Y_UNUSED(settings);
+ }
+ void InitializePQ() {
+ Cerr << "===Init PQ - create server\n";
+ Y_VERIFY(Server == nullptr);
+ PortManager = new TPortManager();
+ Server = MakeHolder<NPersQueue::TTestServer>(false, PortManager);
+ Server->ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(TenantModeEnabled());
+ Server->ServerSettings.PQConfig.SetMetaCacheTimeoutSec(30);
+ AlterSettings(Server->ServerSettings);
+ Server->StartServer(false);
if (TenantModeEnabled()) {
- Server->AnnoyingClient->SetNoConfigMode();
- Server->ServerSettings.PQConfig.SetSourceIdTablePath("some unused path");
+ Server->AnnoyingClient->SetNoConfigMode();
+ Server->ServerSettings.PQConfig.SetSourceIdTablePath("some unused path");
}
- Cerr << "Init PQ - start server on port " << Server->GrpcPort << Endl;
- Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024);
+ Cerr << "Init PQ - start server on port " << Server->GrpcPort << Endl;
+ Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024);
EnablePQLogs({NKikimrServices::PQ_READ_PROXY, NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::FLAT_TX_SCHEMESHARD});
- EnablePQLogs({NKikimrServices::PERSQUEUE}, NLog::EPriority::PRI_INFO);
+ EnablePQLogs({NKikimrServices::PERSQUEUE}, NLog::EPriority::PRI_INFO);
- Server->AnnoyingClient->FullInit();
- Server->AnnoyingClient->CreateConsumer("user");
+ Server->AnnoyingClient->FullInit();
+ Server->AnnoyingClient->CreateConsumer("user");
if (TenantModeEnabled()) {
- Cerr << "=== Will create fst-class topics\n";
- Server->AnnoyingClient->CreateTopicNoLegacy("/Root/acc/topic1", 1);
- Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/acc/topic1", 1);
+ Cerr << "=== Will create fst-class topics\n";
+ Server->AnnoyingClient->CreateTopicNoLegacy("/Root/acc/topic1", 1);
+ Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/acc/topic1", 1);
} else {
- Cerr << "=== Will create legacy-style topics\n";
+ Cerr << "=== Will create legacy-style topics\n";
Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic2dc", 1);
Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc2--acc--topic2dc", 1, true, false);
Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 1);
- Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic1", 1);
- Server->WaitInit("topic1");
- Sleep(TDuration::Seconds(10));
+ Server->AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--acc--topic1", 1);
+ Server->WaitInit("topic1");
+ Sleep(TDuration::Seconds(10));
}
- InsecureChannel = grpc::CreateChannel("localhost:" + ToString(Server->GrpcPort), grpc::InsecureChannelCredentials());
+ InsecureChannel = grpc::CreateChannel("localhost:" + ToString(Server->GrpcPort), grpc::InsecureChannelCredentials());
ServiceStub = Ydb::PersQueue::V1::PersQueueService::NewStub(InsecureChannel);
InitializeWritePQService(TenantModeEnabled() ? "Root/acc/topic1" : "topic1");
@@ -85,7 +85,7 @@ namespace NKikimr::NPersQueueTests {
NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG)
{
for (auto s : services) {
- Server->CleverServer->GetRuntime()->SetLogPriority(s, prio);
+ Server->CleverServer->GetRuntime()->SetLogPriority(s, prio);
}
}
@@ -132,9 +132,9 @@ namespace NKikimr::NPersQueueTests {
void ModifyTopicACL(const TString& topic, const NACLib::TDiffACL& acl) {
if (TenantModeEnabled()) {
TFsPath path(topic);
- Server->AnnoyingClient->ModifyACL(path.Dirname(), path.Basename(), acl.SerializeAsString());
+ Server->AnnoyingClient->ModifyACL(path.Dirname(), path.Basename(), acl.SerializeAsString());
} else {
- Server->AnnoyingClient->ModifyACL("/Root/PQ", topic, acl.SerializeAsString());
+ Server->AnnoyingClient->ModifyACL("/Root/PQ", topic, acl.SerializeAsString());
}
WaitACLModification();
@@ -157,8 +157,8 @@ namespace NKikimr::NPersQueueTests {
}
public:
- THolder<NPersQueue::TTestServer> Server;
- TSimpleSharedPtr<TPortManager> PortManager;
+ THolder<NPersQueue::TTestServer> Server;
+ TSimpleSharedPtr<TPortManager> PortManager;
std::shared_ptr<grpc::Channel> InsecureChannel;
std::unique_ptr<Ydb::PersQueue::V1::PersQueueService::Stub> ServiceStub;
@@ -166,61 +166,61 @@ namespace NKikimr::NPersQueueTests {
THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient;
};
- class TPersQueueV1TestServer : public TPersQueueV1TestServerBase {
+ class TPersQueueV1TestServer : public TPersQueueV1TestServerBase {
public:
- TPersQueueV1TestServer(bool checkAcl = false)
- : CheckACL(checkAcl)
- {
+ TPersQueueV1TestServer(bool checkAcl = false)
+ : CheckACL(checkAcl)
+ {
InitAll();
}
void InitAll() {
- InitializePQ();
+ InitializePQ();
}
-
- void AlterSettings(NKikimr::Tests::TServerSettings& settings) override {
- if (CheckACL)
- settings.PQConfig.SetCheckACL(true);
- }
- private:
- bool CheckACL;
+
+ void AlterSettings(NKikimr::Tests::TServerSettings& settings) override {
+ if (CheckACL)
+ settings.PQConfig.SetCheckACL(true);
+ }
+ private:
+ bool CheckACL;
};
- class TPersQueueV1TestServerWithRateLimiter : public TPersQueueV1TestServerBase {
- private:
- NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity;
+ class TPersQueueV1TestServerWithRateLimiter : public TPersQueueV1TestServerBase {
+ private:
+ NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity;
public:
- TPersQueueV1TestServerWithRateLimiter()
- : TPersQueueV1TestServerBase()
- {}
-
- void AlterSettings(NKikimr::Tests::TServerSettings& settings) override {
- settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true);
- settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity);
- }
-
+ TPersQueueV1TestServerWithRateLimiter()
+ : TPersQueueV1TestServerBase()
+ {}
+
+ void AlterSettings(NKikimr::Tests::TServerSettings& settings) override {
+ settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true);
+ settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity);
+ }
+
void InitAll(NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity limitedEntity) {
- LimitedEntity = limitedEntity;
- InitializePQ();
+ LimitedEntity = limitedEntity;
+ InitializePQ();
InitQuotingPaths();
}
void InitQuotingPaths() {
- Server->AnnoyingClient->MkDir("/Root", "PersQueue");
- Server->AnnoyingClient->MkDir("/Root/PersQueue", "System");
- Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters");
+ Server->AnnoyingClient->MkDir("/Root", "PersQueue");
+ Server->AnnoyingClient->MkDir("/Root/PersQueue", "System");
+ Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters");
}
void CreateTopicWithQuota(const TString& path, bool createKesus = true, double writeQuota = 1000.0) {
TVector<TString> pathComponents = SplitPath(path);
const TString account = pathComponents[0];
- const TString name = NPersQueue::BuildFullTopicName(path, "dc1");
+ const TString name = NPersQueue::BuildFullTopicName(path, "dc1");
if (TenantModeEnabled()) {
- Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/" + path, 1);
+ Server->AnnoyingClient->CreateTopicNoLegacy("/Root/PQ/" + path, 1);
} else {
Cerr << "Creating topic \"" << name << "\"" << Endl;
- Server->AnnoyingClient->CreateTopicNoLegacy(name, 1);
+ Server->AnnoyingClient->CreateTopicNoLegacy(name, 1);
}
const TString rootPath = "/Root/PersQueue/System/Quoters";
@@ -228,12 +228,12 @@ namespace NKikimr::NPersQueueTests {
if (createKesus) {
Cerr << "Creating kesus \"" << account << "\"" << Endl;
- const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(rootPath, account);
+ const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(rootPath, account);
UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult);
- const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
- Server->CleverServer->GetRuntime(), kesusPath, "write-quota", writeQuota
- );
+ const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
+ Server->CleverServer->GetRuntime(), kesusPath, "write-quota", writeQuota
+ );
UNIT_ASSERT_EQUAL_C(statusCode, Ydb::StatusIds::SUCCESS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode));
}
@@ -243,9 +243,9 @@ namespace NKikimr::NPersQueueTests {
for (auto currentComponent = pathComponents.begin() + 1; currentComponent != pathComponents.end(); ++currentComponent) {
prefixPath << "/" << *currentComponent;
Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl;
- const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
- Server->CleverServer->GetRuntime(), kesusPath, prefixPath
- );
+ const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
+ Server->CleverServer->GetRuntime(), kesusPath, prefixPath
+ );
UNIT_ASSERT_C(statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS, "Status: " << Ydb::StatusIds::StatusCode_Name(statusCode));
}
}
@@ -254,7 +254,7 @@ namespace NKikimr::NPersQueueTests {
THolder<IProducer> StartProducer(const TString& topicPath, bool compress = false) {
TString fullPath = TenantModeEnabled() ? "/Root/PQ/" + topicPath : topicPath;
TProducerSettings producerSettings;
- producerSettings.Server = TServerSetting("localhost", Server->GrpcPort);
+ producerSettings.Server = TServerSetting("localhost", Server->GrpcPort);
producerSettings.Topic = fullPath;
producerSettings.SourceId = "TRateLimiterTestSetupSourceId";
producerSettings.Codec = compress ? "gzip" : "raw";
diff --git a/ydb/services/persqueue_v1/ut/pq_data_writer.h b/ydb/services/persqueue_v1/ut/pq_data_writer.h
index 79195ef0f7..caef894060 100644
--- a/ydb/services/persqueue_v1/ut/pq_data_writer.h
+++ b/ydb/services/persqueue_v1/ut/pq_data_writer.h
@@ -13,11 +13,11 @@ namespace NKikimr::NPersQueueTests {
class TPQDataWriter {
public:
- TPQDataWriter(const TString& sourceId, NPersQueue::TTestServer& server, const TString& testTopicPath = "topic1")
+ TPQDataWriter(const TString& sourceId, NPersQueue::TTestServer& server, const TString& testTopicPath = "topic1")
: SourceId_(sourceId)
- , Port_(server.GrpcPort)
- , Client(*server.AnnoyingClient)
- , Runtime(server.CleverServer->GetRuntime())
+ , Port_(server.GrpcPort)
+ , Client(*server.AnnoyingClient)
+ , Runtime(server.CleverServer->GetRuntime())
{
InitializeChannel();
WaitWritePQServiceInitialization(testTopicPath);
@@ -42,7 +42,7 @@ public:
req.mutable_init_request()->add_topics_read_settings()->set_topic(topic);
req.mutable_init_request()->mutable_read_params()->set_max_read_messages_count(1);
- req.mutable_init_request()->set_read_only_original(true);
+ req.mutable_init_request()->set_read_only_original(true);
req.mutable_init_request()->set_consumer(clientId);
@@ -58,7 +58,7 @@ public:
Y_FAIL("");
UNIT_ASSERT(error);
}
- Cerr << "=== Got response: " << resp.ShortDebugString() << Endl;
+ Cerr << "=== Got response: " << resp.ShortDebugString() << Endl;
if (error) {
UNIT_ASSERT(resp.response_case() == Ydb::PersQueue::V1::MigrationStreamingReadServerMessage::RESPONSE_NOT_SET);
return;
diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp
index 08145ddd57..779d60e778 100644
--- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp
+++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp
@@ -6,7 +6,7 @@ using namespace NPersQueue;
using namespace Ydb::PersQueue;
using namespace NKikimr::Tests;
-
+
namespace NKikimr::NPersQueueTests {
TRateLimiterTestSetup::TRateLimiterTestSetup(
@@ -15,7 +15,7 @@ TRateLimiterTestSetup::TRateLimiterTestSetup(
double readAccountQuota,
bool enableReadQuoting
)
- : Server(new NPersQueue::TTestServer(false))
+ : Server(new NPersQueue::TTestServer(false))
, LimitedEntity(limitedEntity)
, WriteAccountQuota(writeAccountQuota)
, ReadAccountQuota(readAccountQuota)
@@ -28,7 +28,7 @@ void TRateLimiterTestSetup::CreateTopic(const TString& path) {
const TString account = GetAccount(name);
Cerr << "Creating topic \"" << name << "\"" << Endl;
- Server->AnnoyingClient->CreateTopic(name, 1);
+ Server->AnnoyingClient->CreateTopic(name, 1);
CreateKesus(account);
CreateQuotaResources(path, "write-quota", false);
@@ -39,7 +39,7 @@ void TRateLimiterTestSetup::CreateConsumer(const TString& path) {
const TString account = GetAccount(path);
Cerr << "Creating consumer \"" << path << "\"" << Endl;
- Server->AnnoyingClient->CreateConsumer(path);
+ Server->AnnoyingClient->CreateConsumer(path);
CreateKesus(account);
CreateQuotaResources(path, "write-quota", true);
@@ -47,7 +47,7 @@ void TRateLimiterTestSetup::CreateConsumer(const TString& path) {
}
void TRateLimiterTestSetup::CreateKesus(const TString& account) {
- const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(QuotersRootPath, account);
+ const NMsgBusProxy::EResponseStatus createKesusResult = Server->AnnoyingClient->CreateKesus(QuotersRootPath, account);
UNIT_ASSERT_C(createKesusResult == NMsgBusProxy::MSTATUS_OK, createKesusResult);
const TString kesusPath = TStringBuilder() << QuotersRootPath << "/" << account;
@@ -55,9 +55,9 @@ void TRateLimiterTestSetup::CreateKesus(const TString& account) {
Cerr << "Creating kesus with path=" << kesusPath << Endl;
auto setAccountQuota = [&](const TString& quotaPrefix, double value) {
Cerr << "Adding quota for account kesus=" << kesusPath << " quota-path=" << quotaPrefix << " value=" << value << Endl;
- const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
- Server->CleverServer->GetRuntime(), kesusPath, quotaPrefix, value
- );
+ const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
+ Server->CleverServer->GetRuntime(), kesusPath, quotaPrefix, value
+ );
UNIT_ASSERT_C(
statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS,
"Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)
@@ -83,9 +83,9 @@ void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TStr
for (auto currentComponent = firstIt; currentComponent != lastIt; ++currentComponent) {
prefixPath << "/" << *currentComponent;
Cerr << "Adding quoter resource: \"" << prefixPath << "\"" << Endl;
- const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
- Server->CleverServer->GetRuntime(), kesusPath, prefixPath
- );
+ const auto statusCode = Server->AnnoyingClient->AddQuoterResource(
+ Server->CleverServer->GetRuntime(), kesusPath, prefixPath
+ );
UNIT_ASSERT_C(
statusCode == Ydb::StatusIds::SUCCESS || statusCode == Ydb::StatusIds::ALREADY_EXISTS,
"Status: " << Ydb::StatusIds::StatusCode_Name(statusCode)
@@ -94,13 +94,13 @@ void TRateLimiterTestSetup::CreateQuotaResources(const TString& path, const TStr
}
/*
-THolder<Ydb::PersQueue::IProducer> TRateLimiterTestSetup::StartProducer(const TString& topicPath, bool compress) {
- Ydb::PersQueue::TProducerSettings producerSettings;
- producerSettings.Server = Ydb::PersQueue::TServerSetting("localhost", Server->GrpcPort);
+THolder<Ydb::PersQueue::IProducer> TRateLimiterTestSetup::StartProducer(const TString& topicPath, bool compress) {
+ Ydb::PersQueue::TProducerSettings producerSettings;
+ producerSettings.Server = Ydb::PersQueue::TServerSetting("localhost", Server->GrpcPort);
producerSettings.Topic = topicPath;
producerSettings.SourceId = "TRateLimiterTestSetupSourceId";
producerSettings.Codec = compress ? "gzip" : "raw";
- THolder<Ydb::PersQueue::IProducer> producer = PQLib->CreateProducer(producerSettings);
+ THolder<Ydb::PersQueue::IProducer> producer = PQLib->CreateProducer(producerSettings);
auto startResult = producer->Start();
UNIT_ASSERT_EQUAL_C(Ydb::StatusIds::SUCCESS, startResult.GetValueSync().Response.status(), "Response: " << startResult.GetValueSync().Response);
return producer;
@@ -114,15 +114,15 @@ void TRateLimiterTestSetup::Start(bool enableReadQuoting) {
}
void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) {
- auto& settings = Server->ServerSettings;
+ auto& settings = Server->ServerSettings;
- settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true);
- settings.PQConfig.MutableQuotingConfig()->SetEnableReadQuoting(enableReadQuoting);
- settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity);
+ settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true);
+ settings.PQConfig.MutableQuotingConfig()->SetEnableReadQuoting(enableReadQuoting);
+ settings.PQConfig.MutableQuotingConfig()->SetTopicWriteQuotaEntityToLimit(LimitedEntity);
- Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024);
- Server->StartServer();
- Server->EnableLogs(
+ Server->GrpcServerOptions.SetMaxMessageSize(130 * 1024 * 1024);
+ Server->StartServer();
+ Server->EnableLogs(
{
NKikimrServices::PQ_READ_PROXY,
NKikimrServices::PQ_WRITE_PROXY,
@@ -137,13 +137,13 @@ void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) {
}
void TRateLimiterTestSetup::InitQuoting() {
- Server->AnnoyingClient->MkDir("/Root", "PersQueue");
- Server->AnnoyingClient->MkDir("/Root/PersQueue", "System");
- Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters");
+ Server->AnnoyingClient->MkDir("/Root", "PersQueue");
+ Server->AnnoyingClient->MkDir("/Root/PersQueue", "System");
+ Server->AnnoyingClient->MkDir("/Root/PersQueue/System", "Quoters");
}
void TRateLimiterTestSetup::WaitWritePQServiceInitialization() {
- PQDataWriter = MakeHolder<TPQDataWriter>("writer_source_id", *Server);
+ PQDataWriter = MakeHolder<TPQDataWriter>("writer_source_id", *Server);
}
}
diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h
index ecd7ce9299..01529818e1 100644
--- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h
+++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.h
@@ -23,19 +23,19 @@ public:
// Getters
ui16 GetGrpcPort() const {
- return Server->GrpcPort;
+ return Server->GrpcPort;
}
ui16 GetMsgBusPort() const {
- return Server->Port;
+ return Server->Port;
}
NKikimr::Tests::TServer& GetServer() {
- return *Server->CleverServer;
+ return *Server->CleverServer;
}
NPersQueueTests::TFlatMsgBusPQClient& GetClient() {
- return *Server->AnnoyingClient;
+ return *Server->AnnoyingClient;
}
private:
@@ -49,7 +49,7 @@ private:
void WaitWritePQServiceInitialization();
private:
- THolder<NPersQueue::TTestServer> Server;
+ THolder<NPersQueue::TTestServer> Server;
THolder<TPQDataWriter> PQDataWriter; // For waiting for grpc writer service initialization.
const NKikimrPQ::TPQConfig::TQuotingConfig::ELimitedEntity LimitedEntity;
double WriteAccountQuota;
diff --git a/ydb/services/persqueue_v1/ut/test_utils.h b/ydb/services/persqueue_v1/ut/test_utils.h
index f4d95edca3..c02308b0d7 100644
--- a/ydb/services/persqueue_v1/ut/test_utils.h
+++ b/ydb/services/persqueue_v1/ut/test_utils.h
@@ -33,7 +33,7 @@ inline void WaitACLModification() {
TServer server = TServer(settings);\
server.EnableGRpc(NGrpc::TServerOptions().SetHost("localhost").SetPort(grpc));\
TFlatMsgBusPQClient client(settings, grpc);\
- client.FullInit();\
+ client.FullInit();\
client.CreateTopic("rt3.dc1--" + TString(topic), 1);\
EnableLogs(server, { NKikimrServices::PQ_WRITE_PROXY });\
TPQDataWriter writer(messageGroupId, grpc, client, server.GetRuntime());\
diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h
index 9aef73f1bd..c067928c78 100644
--- a/ydb/services/ydb/ydb_common_ut.h
+++ b/ydb/services/ydb/ydb_common_ut.h
@@ -80,9 +80,9 @@ public:
ServerSettings->SetEnableYq(enableYq);
ServerSettings->Formats = new TFormatFactory;
ServerSettings->PQConfig = appConfig.GetPQConfig();
- if (appConfig.HasMeteringConfig() && appConfig.GetMeteringConfig().HasMeteringFilePath()) {
- ServerSettings->SetMeteringFilePath(appConfig.GetMeteringConfig().GetMeteringFilePath());
- }
+ if (appConfig.HasMeteringConfig() && appConfig.GetMeteringConfig().HasMeteringFilePath()) {
+ ServerSettings->SetMeteringFilePath(appConfig.GetMeteringConfig().GetMeteringFilePath());
+ }
ServerSettings->RegisterGrpcService<NKikimr::NGRpcService::TGRpcYdbDummyService>("dummy");
if (udfFrFactory) {
ServerSettings->SetFrFactory(udfFrFactory);
diff --git a/ydb/tests/functional/blobstorage/replication.py b/ydb/tests/functional/blobstorage/replication.py
index 49a67190ed..cdf6df6f3f 100644
--- a/ydb/tests/functional/blobstorage/replication.py
+++ b/ydb/tests/functional/blobstorage/replication.py
@@ -1,18 +1,18 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-import pytest
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import pytest
from hamcrest import is_
-
+
from ydb.tests.library.common.types import Erasure
from ydb.tests.library.common.wait_for import wait_for_and_assert
from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
from ydb.tests.library.predicates import blobstorage
-
-
+
+
TIMEOUT_SECONDS = 480
-
-
+
+
class TestReplicationAfterNodesRestart(object):
def __setup_cluster(self, erasure):
self.cluster = kikimr_cluster_factory(configurator=KikimrConfigGenerator(erasure=erasure))
@@ -23,15 +23,15 @@ class TestReplicationAfterNodesRestart(object):
self.cluster.stop()
@pytest.mark.parametrize('erasure', Erasure.common_used(), ids=str)
- def test_replication(self, erasure):
- # Arrange
+ def test_replication(self, erasure):
+ # Arrange
self.__setup_cluster(erasure)
-
- # Act
+
+ # Act
for node in self.cluster.nodes.values():
- node.stop()
- node.start()
-
+ node.stop()
+ node.start()
+
wait_for_and_assert(
lambda: blobstorage.cluster_has_no_unsynced_vdisks(self.cluster),
is_(True), timeout_seconds=TIMEOUT_SECONDS, message='All vdisks are sync'
diff --git a/ydb/tests/functional/blobstorage/ya.make b/ydb/tests/functional/blobstorage/ya.make
index aed9c64738..3248849d57 100644
--- a/ydb/tests/functional/blobstorage/ya.make
+++ b/ydb/tests/functional/blobstorage/ya.make
@@ -1,7 +1,7 @@
OWNER(g:kikimr)
-
+
PY3TEST()
-
+
ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd")
TEST_SRCS(
pdisk_format_info.py
@@ -9,7 +9,7 @@ TEST_SRCS(
self_heal.py
tablet_channel_migration.py
)
-
+
IF (SANITIZER_TYPE)
REQUIREMENTS(
cpu:4
@@ -26,19 +26,19 @@ ELSE()
TIMEOUT(600)
SIZE(MEDIUM)
ENDIF()
-
+
SPLIT_FACTOR(20)
DEPENDS(
ydb/apps/ydbd
)
-
+
PEERDIR(
ydb/tests/library
contrib/python/PyHamcrest
)
-
+
FORK_SUBTESTS()
FORK_TEST_FILES()
-
+
END()
diff --git a/ydb/tests/functional/hive/test_kill_tablets.py b/ydb/tests/functional/hive/test_kill_tablets.py
index 3ed6aa6d0c..08dbd5c0b1 100644
--- a/ydb/tests/functional/hive/test_kill_tablets.py
+++ b/ydb/tests/functional/hive/test_kill_tablets.py
@@ -32,7 +32,7 @@ class TestKillTablets(object):
self.cluster.client.tablet_kill(tablet_id)
generations = {info.TabletId: info.Generation for info in actual_tablet_info}
- wait_tablets_state_by_id(
+ wait_tablets_state_by_id(
self.cluster.client,
TabletStates.Active,
tablet_ids=all_created_tablet_ids,
@@ -57,7 +57,7 @@ class TestKillTablets(object):
is_ok_response()
)
- wait_tablets_state_by_id(
+ wait_tablets_state_by_id(
self.cluster.client,
TabletStates.Active,
tablet_ids=[hive_id],
@@ -67,7 +67,7 @@ class TestKillTablets(object):
all_tablet_ids = create_tablets_and_wait_for_start(self.cluster.client, NUMBER_OF_TABLETS)
# Assert
- wait_tablets_state_by_id(
+ wait_tablets_state_by_id(
self.cluster.client,
TabletStates.Active,
tablet_ids=all_tablet_ids,
diff --git a/ydb/tests/functional/hive/ya.make b/ydb/tests/functional/hive/ya.make
index b5838a7262..60c5fdbf13 100644
--- a/ydb/tests/functional/hive/ya.make
+++ b/ydb/tests/functional/hive/ya.make
@@ -37,7 +37,7 @@ DEPENDS(
PEERDIR(
ydb/tests/library
)
-
+
FORK_SUBTESTS()
FORK_TEST_FILES()
diff --git a/ydb/tests/functional/sqs/sqs_matchers.py b/ydb/tests/functional/sqs/sqs_matchers.py
index 1e0295e168..a8bb8508fc 100644
--- a/ydb/tests/functional/sqs/sqs_matchers.py
+++ b/ydb/tests/functional/sqs/sqs_matchers.py
@@ -1,147 +1,147 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from hamcrest import equal_to, less_than_or_equal_to, contains_inanyorder, contains, greater_than_or_equal_to, all_of, \
- has_item
-from hamcrest.core.base_matcher import BaseMatcher
-
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from hamcrest import equal_to, less_than_or_equal_to, contains_inanyorder, contains, greater_than_or_equal_to, all_of, \
+ has_item
+from hamcrest.core.base_matcher import BaseMatcher
+
from ydb.tests.library.matchers.collection import IsSublistOf
-
-_author__ = 'komels@yandex-team.ru'
-
-
-def extract_field(read_response, field):
- if read_response is None:
- return []
- return [i[field] for i in read_response]
-
-
-def extract_message_ids(read_response):
- return extract_field(read_response, 'MessageId')
-
-
-def extract_message_data(read_response):
- return extract_field(read_response, 'Body')
-
-
-class ReadResponseMatcher(BaseMatcher):
- def __init__(self):
- self.__message_count_matcher = None
- self.__message_ids_matcher = None
- self.__msg_id_to_data = {}
- self.__message_ids = None
-
- self.__messages_data = None
- self.__absent_message_id = None
- self.__description = "read result "
- self.__mismatch_reason = ""
-
- @staticmethod
- def __list_startswith(full_list, sublist):
- for i, v in enumerate(sublist):
- if full_list[i] != v:
- return False
- return True
-
- def with_n_messages(self, messages_count):
- # self.__description += " with {} messages".format(messages_count)
- self.__message_count_matcher = equal_to(messages_count)
- return self
-
- def with_n_of_fewer_messages(self, messages_count):
- # self.__description += " with {} or fewer messages".format(messages_count)
- self.__message_count_matcher = less_than_or_equal_to(messages_count)
- return self
-
+
+_author__ = 'komels@yandex-team.ru'
+
+
+def extract_field(read_response, field):
+ if read_response is None:
+ return []
+ return [i[field] for i in read_response]
+
+
+def extract_message_ids(read_response):
+ return extract_field(read_response, 'MessageId')
+
+
+def extract_message_data(read_response):
+ return extract_field(read_response, 'Body')
+
+
+class ReadResponseMatcher(BaseMatcher):
+ def __init__(self):
+ self.__message_count_matcher = None
+ self.__message_ids_matcher = None
+ self.__msg_id_to_data = {}
+ self.__message_ids = None
+
+ self.__messages_data = None
+ self.__absent_message_id = None
+ self.__description = "read result "
+ self.__mismatch_reason = ""
+
+ @staticmethod
+ def __list_startswith(full_list, sublist):
+ for i, v in enumerate(sublist):
+ if full_list[i] != v:
+ return False
+ return True
+
+ def with_n_messages(self, messages_count):
+ # self.__description += " with {} messages".format(messages_count)
+ self.__message_count_matcher = equal_to(messages_count)
+ return self
+
+ def with_n_of_fewer_messages(self, messages_count):
+ # self.__description += " with {} or fewer messages".format(messages_count)
+ self.__message_count_matcher = less_than_or_equal_to(messages_count)
+ return self
+
def with_n_or_more_messages(self, messages_count):
- self.__message_count_matcher = greater_than_or_equal_to(messages_count)
- return self
-
- def with_message_ids(self, message_ids):
- self.__message_ids_matcher = contains_inanyorder(*message_ids)
- self.__message_ids = message_ids
- return self
-
- def with_some_of_message_ids(self, message_ids):
- self.__message_ids = message_ids
- self.__message_ids_matcher = IsSublistOf(message_ids)
- return self
-
- def with_these_or_more_message_ids(self, message_ids):
- self.__message_ids = message_ids
- self.__message_ids_matcher = all_of(
- *[has_item(m_id) for m_id in message_ids]
- )
- return self
-
- def in_order(self):
- self.__description += " with exact message_id order:\n {}\n"
- if not self.__message_ids:
- raise ValueError("Called in_order, but message_ids to match are not set")
- self.__message_ids_matcher = contains(self.__message_ids)
- return self
-
- def with_messages_data(self, messages_data):
- if self.__message_ids is None:
- raise ValueError("Must set message_ids to verify message data")
- if len(self.__message_ids) != len(messages_data):
- raise ValueError("Must provide message data for same count of messages as message ids")
- for i, msg_id in enumerate(self.__message_ids):
- self.__msg_id_to_data[msg_id] = messages_data[i]
- # self.__description += " with messages containing following data:\n {}\n".format(';'.join(messages_data))
- return self
-
- def without_message_id(self, message_id):
- self.__description += " without message_id: {}".format(message_id)
- self.__absent_message_id = message_id
- return self
-
- def describe_to(self, description):
- description.append_text(self.__description)
- if self.__message_count_matcher is not None:
- description.append_text("with messages count: ")
- self.__message_count_matcher.describe_to(description)
- description.append_text('\n')
- if self.__message_ids_matcher is not None:
- description.append_text("with message_ids: ")
- self.__message_ids_matcher.describe_to(description)
- description.append_text('\n')
- if self.__msg_id_to_data:
- description.append_text("with message_data:\n{}\n".format(self.__msg_id_to_data.values()))
-
- def describe_mismatch(self, actual_response, mismatch_description):
- mismatch_description.append_text("actual result was: messages\n{}\n".format(
- extract_message_ids(actual_response)
- ))
- # message_ids = extract_message_ids(actual_response)
- # if self.__message_count_matcher is not None:
- # self.__message_count_matcher.describe_mismatch(message_ids, mismatch_description)
- # if self.__message_ids_matcher is not None:
- # self.__message_ids_matcher.describe_mismatch(len(message_ids), mismatch_description)
- if self.__msg_id_to_data:
- mismatch_description.append_text("and data: {}\n".format(self.__msg_id_to_data.values()))
- if self.__mismatch_reason:
- mismatch_description.append_text("Mismatch reason was: {}".format(self.__mismatch_reason))
-
- def _matches(self, actual_response):
- actual_message_ids = extract_message_ids(actual_response)
- if len(actual_message_ids) != len(set(actual_message_ids)):
- self.__mismatch_reason += "Duplicate messages appear in result"
- return False
- if self.__message_count_matcher is not None:
- messages_count = len(actual_message_ids)
- if not self.__message_count_matcher.matches(messages_count):
- self.__mismatch_reason += "Messages count didn't match\n"
- return False
- if self.__absent_message_id is not None and self.__absent_message_id is actual_message_ids:
- self.__mismatch_reason += "Message with {} appeared in results"
- return False
- if self.__message_ids_matcher is not None and not self.__message_ids_matcher.matches(actual_message_ids):
- self.__mismatch_reason += "Message ids didn't match"
- return False
- if self.__msg_id_to_data:
- actual_data = extract_message_data(actual_response)
- expected_data = [self.__msg_id_to_data[i] for i in actual_message_ids]
- if sorted(actual_data) != sorted(expected_data):
- self.__mismatch_reason += "Message data didn't match"
- return False
- return True
+ self.__message_count_matcher = greater_than_or_equal_to(messages_count)
+ return self
+
+ def with_message_ids(self, message_ids):
+ self.__message_ids_matcher = contains_inanyorder(*message_ids)
+ self.__message_ids = message_ids
+ return self
+
+ def with_some_of_message_ids(self, message_ids):
+ self.__message_ids = message_ids
+ self.__message_ids_matcher = IsSublistOf(message_ids)
+ return self
+
+ def with_these_or_more_message_ids(self, message_ids):
+ self.__message_ids = message_ids
+ self.__message_ids_matcher = all_of(
+ *[has_item(m_id) for m_id in message_ids]
+ )
+ return self
+
+ def in_order(self):
+ self.__description += " with exact message_id order:\n {}\n"
+ if not self.__message_ids:
+ raise ValueError("Called in_order, but message_ids to match are not set")
+ self.__message_ids_matcher = contains(self.__message_ids)
+ return self
+
+ def with_messages_data(self, messages_data):
+ if self.__message_ids is None:
+ raise ValueError("Must set message_ids to verify message data")
+ if len(self.__message_ids) != len(messages_data):
+ raise ValueError("Must provide message data for same count of messages as message ids")
+ for i, msg_id in enumerate(self.__message_ids):
+ self.__msg_id_to_data[msg_id] = messages_data[i]
+ # self.__description += " with messages containing following data:\n {}\n".format(';'.join(messages_data))
+ return self
+
+ def without_message_id(self, message_id):
+ self.__description += " without message_id: {}".format(message_id)
+ self.__absent_message_id = message_id
+ return self
+
+ def describe_to(self, description):
+ description.append_text(self.__description)
+ if self.__message_count_matcher is not None:
+ description.append_text("with messages count: ")
+ self.__message_count_matcher.describe_to(description)
+ description.append_text('\n')
+ if self.__message_ids_matcher is not None:
+ description.append_text("with message_ids: ")
+ self.__message_ids_matcher.describe_to(description)
+ description.append_text('\n')
+ if self.__msg_id_to_data:
+ description.append_text("with message_data:\n{}\n".format(self.__msg_id_to_data.values()))
+
+ def describe_mismatch(self, actual_response, mismatch_description):
+ mismatch_description.append_text("actual result was: messages\n{}\n".format(
+ extract_message_ids(actual_response)
+ ))
+ # message_ids = extract_message_ids(actual_response)
+ # if self.__message_count_matcher is not None:
+ # self.__message_count_matcher.describe_mismatch(message_ids, mismatch_description)
+ # if self.__message_ids_matcher is not None:
+ # self.__message_ids_matcher.describe_mismatch(len(message_ids), mismatch_description)
+ if self.__msg_id_to_data:
+ mismatch_description.append_text("and data: {}\n".format(self.__msg_id_to_data.values()))
+ if self.__mismatch_reason:
+ mismatch_description.append_text("Mismatch reason was: {}".format(self.__mismatch_reason))
+
+ def _matches(self, actual_response):
+ actual_message_ids = extract_message_ids(actual_response)
+ if len(actual_message_ids) != len(set(actual_message_ids)):
+ self.__mismatch_reason += "Duplicate messages appear in result"
+ return False
+ if self.__message_count_matcher is not None:
+ messages_count = len(actual_message_ids)
+ if not self.__message_count_matcher.matches(messages_count):
+ self.__mismatch_reason += "Messages count didn't match\n"
+ return False
+ if self.__absent_message_id is not None and self.__absent_message_id is actual_message_ids:
+ self.__mismatch_reason += "Message with {} appeared in results"
+ return False
+ if self.__message_ids_matcher is not None and not self.__message_ids_matcher.matches(actual_message_ids):
+ self.__mismatch_reason += "Message ids didn't match"
+ return False
+ if self.__msg_id_to_data:
+ actual_data = extract_message_data(actual_response)
+ expected_data = [self.__msg_id_to_data[i] for i in actual_message_ids]
+ if sorted(actual_data) != sorted(expected_data):
+ self.__mismatch_reason += "Message data didn't match"
+ return False
+ return True
diff --git a/ydb/tests/functional/sqs/sqs_requests_client.py b/ydb/tests/functional/sqs/sqs_requests_client.py
index c32598a4e0..6b7545503f 100644
--- a/ydb/tests/functional/sqs/sqs_requests_client.py
+++ b/ydb/tests/functional/sqs/sqs_requests_client.py
@@ -3,9 +3,9 @@
import itertools
import logging
-import urllib
-import requests
-import xmltodict
+import urllib
+import requests
+import xmltodict
import six
from ydb.tests.library.common.helpers import wrap_in_list
@@ -83,7 +83,7 @@ class SqsHttpApi(object):
self.__auth_headers = auth_headers(user, security_token, iam_token)
if not server.startswith('http'):
server = 'http://' + server
- self.__request = requests.Request(
+ self.__request = requests.Request(
'POST',
"{}:{}".format(server, port),
headers=auth_headers(user, security_token, iam_token)
@@ -93,8 +93,8 @@ class SqsHttpApi(object):
"{}:{}/private".format(server, port),
headers=auth_headers(user, security_token, iam_token)
)
- self.__session = requests.Session()
- self.__raise_on_error = raise_on_error
+ self.__session = requests.Session()
+ self.__raise_on_error = raise_on_error
self.__user = user
self.__timeout = timeout
self.__security_token = security_token
@@ -117,9 +117,9 @@ class SqsHttpApi(object):
request.data = urllib.parse.urlencode(params)
logger.debug("Execute request {} {} from user {} with params: {}".format(
request.method, request.url, self.__user, request.data)
- )
+ )
prep = request.prepare()
- try:
+ try:
response = self.__session.send(prep, timeout=self.__timeout)
except (requests.ConnectionError, requests.exceptions.Timeout) as ex:
logging.debug("Request failed with connection exception {}: {}".format(type(ex), ex))
@@ -127,44 +127,44 @@ class SqsHttpApi(object):
raise
else:
response = None
- return self._process_response(
- response,
- extract_result_method, default
- )
+ return self._process_response(
+ response,
+ extract_result_method, default
+ )
- def _process_response(self, response, extract_method, default=None):
- if response is None:
+ def _process_response(self, response, extract_method, default=None):
+ if response is None:
logging.debug('Returning {} by default'.format(default))
- return default
- if response.status_code != 200:
+ return default
+ if response.status_code != 200:
logger.warn("Last request failed with code {}, reason '{}' and text '{}'".format(
response.status_code, response.reason, response.text
))
# Assert that no internal info will be given to user
assert response.text.find('.cpp:') == -1, 'No internal info should be given to user'
- if self.__raise_on_error:
- raise RuntimeError(
- "Request {} failed with status {} and text {}".format(
- self.__request.data, response.status_code, response.text
- )
- )
- return default
+ if self.__raise_on_error:
+ raise RuntimeError(
+ "Request {} failed with status {} and text {}".format(
+ self.__request.data, response.status_code, response.text
+ )
+ )
+ return default
logging.debug('Parsing response: {}'.format(response.text))
- result = xmltodict.parse(response.text)
+ result = xmltodict.parse(response.text)
try:
- return extract_method(result)
+ return extract_method(result)
except (KeyError, TypeError) as ex:
logger.error("Could not process response from SQS: {}. {}: {}".format(result, type(ex), ex))
- return default
+ return default
- def create_user(self, username):
+ def create_user(self, username):
return self.execute_request(
- action='CreateUser',
- extract_result_method=lambda x: x['CreateUserResponse']['ResponseMetadata'],
- UserName=username
- )
-
+ action='CreateUser',
+ extract_result_method=lambda x: x['CreateUserResponse']['ResponseMetadata'],
+ UserName=username
+ )
+
def delete_user(self, username):
return self.execute_request(
action='DeleteUser',
@@ -172,24 +172,24 @@ class SqsHttpApi(object):
UserName=username
)
- def list_users(self):
+ def list_users(self):
return self.execute_request(
- action='ListUsers',
+ action='ListUsers',
extract_result_method=lambda x: wrap_in_list(x['ListUsersResponse']['ListUsersResult']['UserName'])
- )
-
- def list_queues(self, name_prefix=None):
- if name_prefix is not None:
+ )
+
+ def list_queues(self, name_prefix=None):
+ if name_prefix is not None:
return self.execute_request(
- action='ListQueues',
+ action='ListQueues',
extract_result_method=lambda x: wrap_in_list(x['ListQueuesResponse']['ListQueuesResult']['QueueUrl']), default=(),
- QueueNamePrefix=name_prefix
- )
- else:
+ QueueNamePrefix=name_prefix
+ )
+ else:
return self.execute_request(
- action='ListQueues',
+ action='ListQueues',
extract_result_method=lambda x: wrap_in_list(x['ListQueuesResponse']['ListQueuesResult']['QueueUrl']), default=(),
- )
+ )
def private_count_queues(self):
return self.execute_request(
@@ -198,32 +198,32 @@ class SqsHttpApi(object):
extract_result_method=lambda x: x['CountQueuesResponse']['CountQueuesResult']['Count'],
)
- def create_queue(self, queue_name, is_fifo=False, attributes=None):
- # if is_fifo and not queue_name.endswith('.fifo'):
- # return None
- if attributes is None:
+ def create_queue(self, queue_name, is_fifo=False, attributes=None):
+ # if is_fifo and not queue_name.endswith('.fifo'):
+ # return None
+ if attributes is None:
attributes = dict()
- if is_fifo:
+ if is_fifo:
attributes = dict(attributes) # copy
- attributes['FifoQueue'] = 'true'
- params = {}
- for i, (k, v) in enumerate(attributes.items()):
- params['Attribute.{id}.Name'.format(id=i+1)] = k
- params['Attribute.{id}.Value'.format(id=i + 1)] = v
-
+ attributes['FifoQueue'] = 'true'
+ params = {}
+ for i, (k, v) in enumerate(attributes.items()):
+ params['Attribute.{id}.Name'.format(id=i+1)] = k
+ params['Attribute.{id}.Value'.format(id=i + 1)] = v
+
return self.execute_request(
- action='CreateQueue',
- extract_result_method=lambda x: x['CreateQueueResponse']['CreateQueueResult']['QueueUrl'],
- QueueName=queue_name,
- **params
- )
+ action='CreateQueue',
+ extract_result_method=lambda x: x['CreateQueueResponse']['CreateQueueResult']['QueueUrl'],
+ QueueName=queue_name,
+ **params
+ )
- def delete_queue(self, queue_url):
+ def delete_queue(self, queue_url):
return self.execute_request(
- action='DeleteQueue',
- extract_result_method=lambda x: x['DeleteQueueResponse']['ResponseMetadata']['RequestId'],
- QueueUrl=queue_url
- )
+ action='DeleteQueue',
+ extract_result_method=lambda x: x['DeleteQueueResponse']['ResponseMetadata']['RequestId'],
+ QueueUrl=queue_url
+ )
def private_delete_queue_batch(self, queue_urls):
args = {}
@@ -260,10 +260,10 @@ class SqsHttpApi(object):
def get_queue_url(self, queue_name):
return self.execute_request(
- action='GetQueueUrl',
- extract_result_method=lambda x: x['GetQueueUrlResponse']['GetQueueUrlResult']['QueueUrl'],
- QueueName=queue_name
- )
+ action='GetQueueUrl',
+ extract_result_method=lambda x: x['GetQueueUrlResponse']['GetQueueUrlResult']['QueueUrl'],
+ QueueName=queue_name
+ )
def list_dead_letter_source_queues(self, queue_url):
return self.execute_request(
@@ -383,10 +383,10 @@ class SqsHttpApi(object):
)] = attr.value
return self.execute_request(
- action='SendMessage',
- extract_result_method=lambda x: x['SendMessageResponse']['SendMessageResult']['MessageId'],
- **params
- )
+ action='SendMessage',
+ extract_result_method=lambda x: x['SendMessageResponse']['SendMessageResult']['MessageId'],
+ **params
+ )
def send_message_batch(self, queue_url, send_message_params_list):
params = {
@@ -470,31 +470,31 @@ class SqsHttpApi(object):
params['ReceiveRequestAttemptId'] = receive_request_attempt_id
return self.execute_request(
- action='ReceiveMessage',
- extract_result_method=lambda x: wrap_in_list(
- x['ReceiveMessageResponse']['ReceiveMessageResult']['Message']
- ),
- **params
+ action='ReceiveMessage',
+ extract_result_method=lambda x: wrap_in_list(
+ x['ReceiveMessageResponse']['ReceiveMessageResult']['Message']
+ ),
+ **params
)
def delete_message(self, queue_url, handle):
return self.execute_request(
- action='DeleteMessage',
- extract_result_method=lambda x: x['DeleteMessageResponse']['ResponseMetadata']['RequestId'],
- QueueUrl=queue_url, ReceiptHandle=handle
- )
-
- def delete_message_batch(self, queue_url, message_handles):
- args = {}
- for i, handle in enumerate(message_handles):
+ action='DeleteMessage',
+ extract_result_method=lambda x: x['DeleteMessageResponse']['ResponseMetadata']['RequestId'],
+ QueueUrl=queue_url, ReceiptHandle=handle
+ )
+
+ def delete_message_batch(self, queue_url, message_handles):
+ args = {}
+ for i, handle in enumerate(message_handles):
args["DeleteMessageBatchRequestEntry.{}.Id".format(i+1)] = str(i)
- args["DeleteMessageBatchRequestEntry.{}.ReceiptHandle".format(i+1)] = handle
-
+ args["DeleteMessageBatchRequestEntry.{}.ReceiptHandle".format(i+1)] = handle
+
resp = self.execute_request(
- action='DeleteMessageBatch',
+ action='DeleteMessageBatch',
extract_result_method=lambda x: x['DeleteMessageBatchResponse'],
- QueueUrl=queue_url, **args
- )
+ QueueUrl=queue_url, **args
+ )
result = [dict() for i in six.moves.range(len(message_handles))]
results = resp.get('DeleteMessageBatchResult')
logging.debug('results: {}'.format(results))
@@ -506,7 +506,7 @@ class SqsHttpApi(object):
i = int(res['Id'])
assert i < len(result) and i >= 0
result[i]['DeleteMessageBatchResultEntry'] = res
-
+
errors = results.get('BatchResultErrorEntry')
logging.debug('errors: {}'.format(errors))
if errors:
@@ -515,7 +515,7 @@ class SqsHttpApi(object):
assert i < len(result) and i >= 0
result[i]['BatchResultErrorEntry'] = err
return result
-
+
def change_message_visibility(self, queue_url, handle, visibility_timeout):
return self.execute_request(
action='ChangeMessageVisibility',
@@ -557,19 +557,19 @@ class SqsHttpApi(object):
return result
-class SqsHttpMinigunApi(SqsHttpApi):
- def _process_response(self, response, extract_method, default=None):
- if response is None:
- return -1, default
- try:
- parsed = xmltodict.parse(response.text)
- except xmltodict.expat.ExpatError:
- return response.status_code, default
-
- if response.status_code != 200:
- extract_method = lambda x: x['ErrorResponse']['Error']['Message']
- try:
- result = extract_method(parsed)
- except (KeyError, TypeError):
- result = default
- return response.status_code, result
+class SqsHttpMinigunApi(SqsHttpApi):
+ def _process_response(self, response, extract_method, default=None):
+ if response is None:
+ return -1, default
+ try:
+ parsed = xmltodict.parse(response.text)
+ except xmltodict.expat.ExpatError:
+ return response.status_code, default
+
+ if response.status_code != 200:
+ extract_method = lambda x: x['ErrorResponse']['Error']['Message']
+ try:
+ result = extract_method(parsed)
+ except (KeyError, TypeError):
+ result = default
+ return response.status_code, result
diff --git a/ydb/tests/functional/sqs/sqs_test_base.py b/ydb/tests/functional/sqs/sqs_test_base.py
index 12b98ad1b8..05d5303f64 100644
--- a/ydb/tests/functional/sqs/sqs_test_base.py
+++ b/ydb/tests/functional/sqs/sqs_test_base.py
@@ -24,7 +24,7 @@ from concurrent import futures
from sqs_matchers import ReadResponseMatcher
-DEFAULT_VISIBILITY_TIMEOUT = 30
+DEFAULT_VISIBILITY_TIMEOUT = 30
logger = logging.getLogger(__name__)
@@ -138,15 +138,15 @@ def get_test_with_sqs_installation_by_path(base_test_class):
return TestWithPath
-class KikimrSqsTestBase(object):
+class KikimrSqsTestBase(object):
erasure = None
slot_count = 0
database = '/Root'
sqs_root = '/Root/SQS'
use_in_memory_pdisks = True
-
- @classmethod
- def setup_class(cls):
+
+ @classmethod
+ def setup_class(cls):
cls.cluster, cls.config_generator = cls._setup_cluster()
cls.sqs_ports = []
if cls.slot_count:
@@ -158,8 +158,8 @@ class KikimrSqsTestBase(object):
cls.sqs_ports.append(node.sqs_port)
cls.sqs_port = cls.sqs_ports[0]
- cls.server_fqdn = get_fqdn()
-
+ cls.server_fqdn = get_fqdn()
+
def setup_method(self, method=None):
logging.debug('Test started: {}'.format(str(method.__name__)))
logging.debug("Kikimr logs dir: {}".format(self.cluster.slots[1].cwd if self.slot_count else self.cluster.nodes[1].cwd))
@@ -216,8 +216,8 @@ class KikimrSqsTestBase(object):
self.message_ids = []
self.read_result = []
- self.seq_no = 0
-
+ self.seq_no = 0
+
def teardown_method(self, method=None):
self.check_no_queues_table(self._username)
self._driver.stop()
@@ -246,14 +246,14 @@ class KikimrSqsTestBase(object):
assert_that(raised)
- @classmethod
- def teardown_class(cls):
- if hasattr(cls, 'cluster'):
- cls.cluster.stop()
-
+ @classmethod
+ def teardown_class(cls):
+ if hasattr(cls, 'cluster'):
+ cls.cluster.stop()
+
@classmethod
def _setup_config_generator(cls):
- config_generator = KikimrConfigGenerator(
+ config_generator = KikimrConfigGenerator(
erasure=cls.erasure,
use_in_memory_pdisks=cls.use_in_memory_pdisks,
additional_log_configs={'SQS': LogLevels.INFO},
@@ -268,7 +268,7 @@ class KikimrSqsTestBase(object):
config_generator.yaml_config['sqs_config']['check_all_shards_in_receive_message'] = True
config_generator.yaml_config['sqs_config']['create_legacy_duration_counters'] = False
config_generator.yaml_config['sqs_config']['validate_message_body'] = True
-
+
return config_generator
@classmethod
@@ -313,17 +313,17 @@ class KikimrSqsTestBase(object):
'-u', 'metauser',
'-n', _username,
] + self._sqs_server_opts
- while retries_count:
+ while retries_count:
logging.debug("Running {}".format(' '.join(cmd)))
- try:
+ try:
yatest_common.execute(cmd)
except yatest_common.ExecutionError as ex:
logging.debug("Create user failed: {}. Retrying".format(ex))
- retries_count -= 1
+ retries_count -= 1
time.sleep(3)
- else:
+ else:
return
- raise RuntimeError("Failed to create SQS user")
+ raise RuntimeError("Failed to create SQS user")
def _create_api_for_user(self, user_name, raise_on_error=True, security_token=None, force_private=False, iam_token=None, folder_id=None):
api = SqsHttpApi(self.cluster.nodes[1].host,
@@ -368,7 +368,7 @@ class KikimrSqsTestBase(object):
else:
raise
- def _queue_url_matcher(self, queue_name):
+ def _queue_url_matcher(self, queue_name):
urls_matchers = [
equal_to(
to_bytes(
@@ -388,9 +388,9 @@ class KikimrSqsTestBase(object):
logging.debug('Create queue. Attributes: {}. Use http: {}. Is fifo: {}'.format(attributes, use_http, is_fifo))
assert (len(attributes.keys()) == 0 or use_http), 'Attributes are supported only for http queue creation'
assert (shards is None or not use_http), 'Custom shards number is only supported in non-http mode'
- while retries:
+ while retries:
retries -= 1
- try:
+ try:
if use_http:
self.queue_url = self._sqs_api.create_queue(queue_name, is_fifo=is_fifo, attributes=attributes)
else:
@@ -412,52 +412,52 @@ class KikimrSqsTestBase(object):
else:
raise
if self.queue_url is not None: # queue_url will be None in case of connection error
- break
+ break
assert_that(
to_bytes(self.queue_url),
- self._queue_url_matcher(queue_name)
+ self._queue_url_matcher(queue_name)
)
return self.queue_url
def _send_message_and_assert(self, queue_url, msg_body, seq_no=None, group_id=None, attributes=None, delay_seconds=None):
attributes = {} if attributes is None else attributes
- send_msg_result = self._sqs_api.send_message(
+ send_msg_result = self._sqs_api.send_message(
queue_url, msg_body, deduplication_id=seq_no, group_id=group_id, attributes=attributes, delay_seconds=delay_seconds
- )
+ )
assert_that(
send_msg_result, not_none()
)
return send_msg_result
- def _send_messages(self, queue_url, message_count, msg_body_template=None, is_fifo=False, group_id=None):
- if msg_body_template is None:
- msg_body_template = self._msg_body_template
+ def _send_messages(self, queue_url, message_count, msg_body_template=None, is_fifo=False, group_id=None):
+ if msg_body_template is None:
+ msg_body_template = self._msg_body_template
ret = []
for _ in range(message_count):
- if is_fifo:
- result = self._send_message_and_assert(
+ if is_fifo:
+ result = self._send_message_and_assert(
queue_url, msg_body_template.format(next(self.counter)), seq_no=self.seq_no, group_id=group_id
- )
- self.seq_no += 1
- else:
+ )
+ self.seq_no += 1
+ else:
result = self._send_message_and_assert(queue_url, msg_body_template.format(next(self.counter)))
ret.append(result)
return ret
def _read_while_not_empty(self, queue_url, messages_count, visibility_timeout=None, wait_timeout=1, max_empty_reads=1):
- ret = []
- messages_by_time = {}
- actual_vis_timeout = visibility_timeout if visibility_timeout is not None else DEFAULT_VISIBILITY_TIMEOUT
+ ret = []
+ messages_by_time = {}
+ actual_vis_timeout = visibility_timeout if visibility_timeout is not None else DEFAULT_VISIBILITY_TIMEOUT
empty_reads_count = 0
max_batch_to_read = self.config_generator.yaml_config['sqs_config']['max_number_of_receive_messages']
- while len(ret) < messages_count:
- # noinspection PyTypeChecker
+ while len(ret) < messages_count:
+ # noinspection PyTypeChecker
request_start = time.time()
- read_result = self._sqs_api.receive_message(
+ read_result = self._sqs_api.receive_message(
queue_url, max_number_of_messages=min(messages_count - len(ret), max_batch_to_read),
visibility_timeout=visibility_timeout, wait_timeout=wait_timeout
- )
- if not read_result:
+ )
+ if not read_result:
empty_reads_count += 1
if empty_reads_count == max_empty_reads:
break
@@ -477,38 +477,38 @@ class KikimrSqsTestBase(object):
)
else:
raise AssertionError("Message {} appeared twice before visibility timeout expired".format(msg_id))
- return ret
-
- def _read_messages_and_assert(
+ return ret
+
+ def _read_messages_and_assert(
self, queue_url, messages_count, matcher=None, visibility_timeout=None, wait_timeout=1
):
read_result = self._read_while_not_empty(
- queue_url, messages_count=messages_count,
+ queue_url, messages_count=messages_count,
visibility_timeout=visibility_timeout, wait_timeout=wait_timeout
)
- if matcher is not None:
- assert_that(
- read_result, matcher
- )
+ if matcher is not None:
+ assert_that(
+ read_result, matcher
+ )
return read_result
- def _create_queue_send_x_messages_read_y_messages(
- self, queue_name, send_count, read_count, msg_body_template,
+ def _create_queue_send_x_messages_read_y_messages(
+ self, queue_name, send_count, read_count, msg_body_template,
is_fifo=False, visibility_timeout=None, wait_timeout=1, group_id="1"
):
self._create_queue_and_assert(queue_name, is_fifo)
if is_fifo:
- self.message_ids = self._send_messages(
- self.queue_url, send_count, msg_body_template, is_fifo=True, group_id=group_id
- )
- else:
- self.message_ids = self._send_messages(
- self.queue_url, send_count, msg_body_template
- )
- self.read_result = self._read_messages_and_assert(
- self.queue_url, read_count,
- ReadResponseMatcher().with_some_of_message_ids(self.message_ids).with_n_messages(read_count),
- visibility_timeout, wait_timeout
+ self.message_ids = self._send_messages(
+ self.queue_url, send_count, msg_body_template, is_fifo=True, group_id=group_id
+ )
+ else:
+ self.message_ids = self._send_messages(
+ self.queue_url, send_count, msg_body_template
+ )
+ self.read_result = self._read_messages_and_assert(
+ self.queue_url, read_count,
+ ReadResponseMatcher().with_some_of_message_ids(self.message_ids).with_n_messages(read_count),
+ visibility_timeout, wait_timeout
)
def _other_node(self, node_index):
@@ -516,7 +516,7 @@ class KikimrSqsTestBase(object):
return 1
else:
return node_index - 1
-
+
def _get_live_node_index(self):
for i in range(self.cluster_nodes_count):
if self.slot_count:
@@ -534,12 +534,12 @@ class KikimrSqsTestBase(object):
return self.config_generator.port_allocator.get_node_port_allocator(node_index + 1).mon_port
def _get_sqs_counters(self, node_index=0, counters_format='json'):
- return self._get_counters(node_index, "sqs", counters_format)
-
- def _get_ymq_counters(self, cloud, folder, node_index=0, counters_format='json'):
- return self._get_counters(node_index, "ymq_public", counters_format, cloud=cloud, folder=folder)
-
- def _get_counters(self, node_index, component, counters_format, cloud=None, folder=None):
+ return self._get_counters(node_index, "sqs", counters_format)
+
+ def _get_ymq_counters(self, cloud, folder, node_index=0, counters_format='json'):
+ return self._get_counters(node_index, "ymq_public", counters_format, cloud=cloud, folder=folder)
+
+ def _get_counters(self, node_index, component, counters_format, cloud=None, folder=None):
mon_port = self._get_mon_port(node_index)
if counters_format == 'json':
@@ -549,13 +549,13 @@ class KikimrSqsTestBase(object):
else:
raise Exception('Unknown counters format: \"{}\"'.format(counters_format))
- if folder is not None:
- labels="/cloud%3D{cloud}/folder%3D{folder}".format(cloud=cloud, folder=folder)
- else:
- labels = ''
- counters_url = 'http://localhost:{port}/counters/counters%3D{component}{labels}{suffix}'.format(
- port=mon_port, component=component, suffix=format_suffix, labels=labels
- )
+ if folder is not None:
+ labels="/cloud%3D{cloud}/folder%3D{folder}".format(cloud=cloud, folder=folder)
+ else:
+ labels = ''
+ counters_url = 'http://localhost:{port}/counters/counters%3D{component}{labels}{suffix}'.format(
+ port=mon_port, component=component, suffix=format_suffix, labels=labels
+ )
reply = requests.get(counters_url)
assert_that(reply.status_code, equal_to(200))
diff --git a/ydb/tests/functional/sqs/test_fifo_messaging.py b/ydb/tests/functional/sqs/test_fifo_messaging.py
index 1fd35885e7..6410b21966 100644
--- a/ydb/tests/functional/sqs/test_fifo_messaging.py
+++ b/ydb/tests/functional/sqs/test_fifo_messaging.py
@@ -3,10 +3,10 @@
import logging
import time
-import pytest
+import pytest
from hamcrest import assert_that, equal_to, not_none, greater_than, less_than_or_equal_to, has_items, raises
-from sqs_matchers import ReadResponseMatcher, extract_message_ids
+from sqs_matchers import ReadResponseMatcher, extract_message_ids
from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, VISIBILITY_CHANGE_METHOD_PARAMS
@@ -41,41 +41,41 @@ class TestSqsFifoMicroBatchesWithPath(get_test_with_sqs_installation_by_path(Sqs
class SqsFifoMessagingTest(KikimrSqsTestBase):
- def setup_method(self, method=None):
+ def setup_method(self, method=None):
super(SqsFifoMessagingTest, self).setup_method(method)
- self.queue_name = self.queue_name + ".fifo"
-
+ self.queue_name = self.queue_name + ".fifo"
+
@classmethod
def _setup_config_generator(cls):
config_generator = super(SqsFifoMessagingTest, cls)._setup_config_generator()
config_generator.yaml_config['sqs_config']['group_selection_batch_size'] = 100
return config_generator
- def test_only_single_read_infly_from_fifo(self):
- self._create_queue_send_x_messages_read_y_messages(
+ def test_only_single_read_infly_from_fifo(self):
+ self._create_queue_send_x_messages_read_y_messages(
self.queue_name, send_count=10, read_count=1, visibility_timeout=1000,
msg_body_template=self._msg_body_template, is_fifo=True
- )
- self._read_messages_and_assert(
+ )
+ self._read_messages_and_assert(
self.queue_url, messages_count=10, matcher=ReadResponseMatcher().with_n_messages(0)
- )
-
- def test_fifo_read_delete_single_message(self):
- created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
- message_ids = self._send_messages(
+ )
+
+ def test_fifo_read_delete_single_message(self):
+ created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
+ message_ids = self._send_messages(
created_queue_url, message_count=10, msg_body_template=self._msg_body_template, is_fifo=True, group_id='group'
- )
- read_result = self._read_messages_and_assert(
+ )
+ read_result = self._read_messages_and_assert(
self.queue_url, messages_count=1, matcher=ReadResponseMatcher().with_message_ids(message_ids[:1])
- )
- handle = read_result[0]['ReceiptHandle']
- assert_that(
- self._sqs_api.delete_message(self.queue_url, handle), not_none()
- )
- self._read_messages_and_assert(
+ )
+ handle = read_result[0]['ReceiptHandle']
+ assert_that(
+ self._sqs_api.delete_message(self.queue_url, handle), not_none()
+ )
+ self._read_messages_and_assert(
self.queue_url, messages_count=1, matcher=ReadResponseMatcher().with_message_ids(message_ids[1:2])
- )
-
+ )
+
counters = self._get_sqs_counters()
delete_counter_labels = {
'subsystem': 'core',
@@ -85,129 +85,129 @@ class SqsFifoMessagingTest(KikimrSqsTestBase):
}
assert_that(self._get_counter_value(counters, delete_counter_labels, 0), equal_to(1))
- def test_write_and_read_to_different_groups(self):
- seq_no = 1
- self._create_queue_and_assert(self.queue_name, is_fifo=True)
- message_ids = []
- for group_id in range(10):
- msg_id = self._send_message_and_assert(
- self.queue_url, self._msg_body_template.format('1'), seq_no, str(group_id)
- )
- seq_no += 1
- message_ids.append(msg_id)
-
- result = self._read_messages_and_assert(
+ def test_write_and_read_to_different_groups(self):
+ seq_no = 1
+ self._create_queue_and_assert(self.queue_name, is_fifo=True)
+ message_ids = []
+ for group_id in range(10):
+ msg_id = self._send_message_and_assert(
+ self.queue_url, self._msg_body_template.format('1'), seq_no, str(group_id)
+ )
+ seq_no += 1
+ message_ids.append(msg_id)
+
+ result = self._read_messages_and_assert(
self.queue_url, 10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_n_messages(10)
- )
- received_message_ids = extract_message_ids(result)
- assert_that(
- sorted(received_message_ids), equal_to(sorted(message_ids))
- )
-
- def test_can_read_from_different_groups(self):
- seq_no = 1
- self._create_queue_and_assert(self.queue_name, is_fifo=True)
- message_ids = []
- for group_id in range(20):
- msg_id = self._send_message_and_assert(
- self.queue_url, self._msg_body_template.format(group_id), seq_no, str(group_id)
- )
- seq_no += 1
- message_ids.append(msg_id)
-
- first_message_ids = extract_message_ids(
- self._read_messages_and_assert(
+ )
+ received_message_ids = extract_message_ids(result)
+ assert_that(
+ sorted(received_message_ids), equal_to(sorted(message_ids))
+ )
+
+ def test_can_read_from_different_groups(self):
+ seq_no = 1
+ self._create_queue_and_assert(self.queue_name, is_fifo=True)
+ message_ids = []
+ for group_id in range(20):
+ msg_id = self._send_message_and_assert(
+ self.queue_url, self._msg_body_template.format(group_id), seq_no, str(group_id)
+ )
+ seq_no += 1
+ message_ids.append(msg_id)
+
+ first_message_ids = extract_message_ids(
+ self._read_messages_and_assert(
self.queue_url, 10, ReadResponseMatcher().with_n_messages(10), visibility_timeout=1000
- )
- )
- second_message_ids = extract_message_ids(
- self._read_messages_and_assert(
+ )
+ )
+ second_message_ids = extract_message_ids(
+ self._read_messages_and_assert(
self.queue_url, 10, ReadResponseMatcher().with_n_messages(10), visibility_timeout=1000
- )
- )
-
- assert_that(
- len(set(first_message_ids + second_message_ids)),
- equal_to(len(first_message_ids) + len(second_message_ids))
- )
- self._read_messages_and_assert(
+ )
+ )
+
+ assert_that(
+ len(set(first_message_ids + second_message_ids)),
+ equal_to(len(first_message_ids) + len(second_message_ids))
+ )
+ self._read_messages_and_assert(
self.queue_url, 10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_n_messages(0)
- )
-
- def test_send_and_read_multiple_messages(self):
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
- first_message_id = self._send_message_and_assert(
+ )
+
+ def test_send_and_read_multiple_messages(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
+ first_message_id = self._send_message_and_assert(
queue_url, self._msg_body_template.format('0'), seq_no=1, group_id='group'
- )
+ )
time.sleep(5)
- self._send_message_and_assert(
+ self._send_message_and_assert(
queue_url, self._msg_body_template.format('1'), seq_no=2, group_id='group'
- )
- self._read_messages_and_assert(
+ )
+ self._read_messages_and_assert(
queue_url, messages_count=1,
- matcher=ReadResponseMatcher().with_message_ids([first_message_id, ])
- )
-
- def test_read_dont_stall(self):
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
- pack_size = 5
- first_pack_ids = self._send_messages(
- queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True,
- group_id='1',
- )
+ matcher=ReadResponseMatcher().with_message_ids([first_message_id, ])
+ )
+
+ def test_read_dont_stall(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
+ pack_size = 5
+ first_pack_ids = self._send_messages(
+ queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True,
+ group_id='1',
+ )
time.sleep(5)
- second_pack_ids = self._send_messages(
- queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True,
- group_id='2'
- )
+ second_pack_ids = self._send_messages(
+ queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True,
+ group_id='2'
+ )
time.sleep(5)
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
queue_url, messages_count=10, visibility_timeout=1000,
- matcher=ReadResponseMatcher().with_message_ids(
- [first_pack_ids[0], second_pack_ids[0]]
- )
- )
- third_pack_ids = self._send_messages(
- queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True,
- group_id='3'
- )
- self._read_messages_and_assert(
+ matcher=ReadResponseMatcher().with_message_ids(
+ [first_pack_ids[0], second_pack_ids[0]]
+ )
+ )
+ third_pack_ids = self._send_messages(
+ queue_url, message_count=pack_size, msg_body_template=self._msg_body_template, is_fifo=True,
+ group_id='3'
+ )
+ self._read_messages_and_assert(
queue_url, messages_count=10, visibility_timeout=1000,
- matcher=ReadResponseMatcher().with_message_ids(third_pack_ids[:1])
- )
-
- def test_visibility_timeout_works(self):
- self._create_queue_send_x_messages_read_y_messages(
- self.queue_name, send_count=5, read_count=1, visibility_timeout=10,
+ matcher=ReadResponseMatcher().with_message_ids(third_pack_ids[:1])
+ )
+
+ def test_visibility_timeout_works(self):
+ self._create_queue_send_x_messages_read_y_messages(
+ self.queue_name, send_count=5, read_count=1, visibility_timeout=10,
msg_body_template=self._msg_body_template, is_fifo=True, group_id='1'
- )
- second_pack_ids = self._send_messages(self.queue_url, 5, group_id='2', is_fifo=True)
- self._read_messages_and_assert(
+ )
+ second_pack_ids = self._send_messages(self.queue_url, 5, group_id='2', is_fifo=True)
+ self._read_messages_and_assert(
self.queue_url, messages_count=5, matcher=ReadResponseMatcher().with_these_or_more_message_ids(second_pack_ids[:1]),
- visibility_timeout=10
- )
+ visibility_timeout=10
+ )
time.sleep(12)
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
self.queue_url, messages_count=5, visibility_timeout=1000, matcher=ReadResponseMatcher().with_these_or_more_message_ids(
- [self.message_ids[0], second_pack_ids[0]]
- )
- )
-
- def test_delete_message_works(self):
- self._create_queue_send_x_messages_read_y_messages(
- self.queue_name, send_count=10, read_count=1, visibility_timeout=1,
+ [self.message_ids[0], second_pack_ids[0]]
+ )
+ )
+
+ def test_delete_message_works(self):
+ self._create_queue_send_x_messages_read_y_messages(
+ self.queue_name, send_count=10, read_count=1, visibility_timeout=1,
msg_body_template=self._msg_body_template, is_fifo=True
- )
-
- handle = self.read_result[0]['ReceiptHandle']
- assert_that(
- self._sqs_api.delete_message(self.queue_url, handle), not_none()
- )
+ )
+
+ handle = self.read_result[0]['ReceiptHandle']
+ assert_that(
+ self._sqs_api.delete_message(self.queue_url, handle), not_none()
+ )
time.sleep(1)
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
self.queue_url, messages_count=5, visibility_timeout=1000, matcher=ReadResponseMatcher().with_message_ids(self.message_ids[1:2])
- )
-
+ )
+
counters = self._get_sqs_counters()
delete_counter_labels = {
'subsystem': 'core',
@@ -230,32 +230,32 @@ class SqsFifoMessagingTest(KikimrSqsTestBase):
raises(RuntimeError, pattern='InternalFailure')
)
- def test_write_read_delete_many_groups(self):
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
- message_ids = {}
- for i in range(10):
- message_ids[i] = self._send_messages(queue_url, 5, is_fifo=True, group_id=str(i))
-
- matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids(
- [i[0] for i in message_ids.values()]
- ).with_messages_data(
- [self._msg_body_template.format(5*i) for i in range(10)]
- )
- result = self._read_messages_and_assert(queue_url, 10, matcher=matcher, visibility_timeout=5)
- # Delete message from group 0
- for msg in result:
- if msg['MessageId'] in message_ids[0]:
- assert_that(
- self._sqs_api.delete_message(self.queue_url, msg['ReceiptHandle']), not_none()
- )
- message_ids[0] = message_ids[0][1:]
- break
+ def test_write_read_delete_many_groups(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=True)
+ message_ids = {}
+ for i in range(10):
+ message_ids[i] = self._send_messages(queue_url, 5, is_fifo=True, group_id=str(i))
+
+ matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids(
+ [i[0] for i in message_ids.values()]
+ ).with_messages_data(
+ [self._msg_body_template.format(5*i) for i in range(10)]
+ )
+ result = self._read_messages_and_assert(queue_url, 10, matcher=matcher, visibility_timeout=5)
+ # Delete message from group 0
+ for msg in result:
+ if msg['MessageId'] in message_ids[0]:
+ assert_that(
+ self._sqs_api.delete_message(self.queue_url, msg['ReceiptHandle']), not_none()
+ )
+ message_ids[0] = message_ids[0][1:]
+ break
time.sleep(5)
- matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids(
- [i[0] for i in message_ids.values()]
- ).with_messages_data(
- [self._msg_body_template.format(1)] + [self._msg_body_template.format(i*5) for i in range(1, 10)]
- )
+ matcher = ReadResponseMatcher().with_n_messages(10).with_message_ids(
+ [i[0] for i in message_ids.values()]
+ ).with_messages_data(
+ [self._msg_body_template.format(1)] + [self._msg_body_template.format(i*5) for i in range(1, 10)]
+ )
self._read_messages_and_assert(queue_url, 10, matcher=matcher, visibility_timeout=1000)
def test_queue_attributes(self):
diff --git a/ydb/tests/functional/sqs/test_garbage_collection.py b/ydb/tests/functional/sqs/test_garbage_collection.py
index bc09e02951..992c8a6513 100644
--- a/ydb/tests/functional/sqs/test_garbage_collection.py
+++ b/ydb/tests/functional/sqs/test_garbage_collection.py
@@ -5,7 +5,7 @@ import time
import multiprocessing
import random
-import pytest
+import pytest
from hamcrest import assert_that, equal_to, less_than_or_equal_to
from sqs_requests_client import SqsHttpApi
diff --git a/ydb/tests/functional/sqs/test_generic_messaging.py b/ydb/tests/functional/sqs/test_generic_messaging.py
index 8ea67f99dd..3f6b2371ba 100644
--- a/ydb/tests/functional/sqs/test_generic_messaging.py
+++ b/ydb/tests/functional/sqs/test_generic_messaging.py
@@ -5,22 +5,22 @@ import logging
import time
from collections import OrderedDict
-import pytest
+import pytest
from hamcrest import assert_that, equal_to, not_none, greater_than, has_item, has_items, raises, is_not, not_, empty, instance_of
from sqs_requests_client import SqsMessageAttribute, SqsSendMessageParams, SqsChangeMessageVisibilityParams
-from sqs_matchers import ReadResponseMatcher, extract_message_ids
+from sqs_matchers import ReadResponseMatcher, extract_message_ids
from sqs_test_base import to_bytes
from sqs_test_base import KikimrSqsTestBase, get_test_with_sqs_installation_by_path, get_test_with_sqs_tenant_installation, IS_FIFO_PARAMS
class SqsGenericMessagingTest(KikimrSqsTestBase):
@pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_send_message(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
- created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
+ def test_send_message(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+ created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
self.seq_no += 1
self._send_message_and_assert(created_queue_url, 'test_send_message', seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None)
@@ -130,11 +130,11 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
)
@pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_send_and_read_message(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
-
- created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
+ def test_send_and_read_message(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+
+ created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
body = '<' + self._msg_body_template.format('trololo') + '<' # to ensure that we have correct xml
attributes = {
SqsMessageAttribute('a', 'String', 'xyz'),
@@ -304,89 +304,89 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
)
@pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_create_q_twice(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
- created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
+ def test_create_q_twice(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+ created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
self.seq_no += 1
message_id = self._send_message_and_assert(created_queue_url, self._msg_body_template, seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None)
- second_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
+ second_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
assert_that(second_url, equal_to(created_queue_url))
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
created_queue_url,
messages_count=1,
visibility_timeout=1000, matcher=ReadResponseMatcher().with_message_ids([message_id, ])
- )
+ )
- def test_send_and_read_multiple_messages(self):
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False)
- first_message_id = self._send_message_and_assert(
- queue_url, self._msg_body_template.format('0')
- )
+ def test_send_and_read_multiple_messages(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False)
+ first_message_id = self._send_message_and_assert(
+ queue_url, self._msg_body_template.format('0')
+ )
time.sleep(5)
- second_message_id = self._send_message_and_assert(
- queue_url, self._msg_body_template.format('1')
- )
- self._read_messages_and_assert(
+ second_message_id = self._send_message_and_assert(
+ queue_url, self._msg_body_template.format('1')
+ )
+ self._read_messages_and_assert(
queue_url, messages_count=2, visibility_timeout=1000,
- matcher=ReadResponseMatcher().with_message_ids(
- [first_message_id, second_message_id]
- ).with_messages_data([self._msg_body_template.format('0'), self._msg_body_template.format('1')])
- )
-
- def test_read_dont_stall(self):
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False)
- first_pack_size = 5
- second_pack_size = 5
- first_pack_ids = self._send_messages(
- queue_url, message_count=first_pack_size, msg_body_template=self._msg_body_template
- )
- self._send_messages(queue_url, message_count=second_pack_size, msg_body_template=self._msg_body_template)
- self._read_messages_and_assert(
+ matcher=ReadResponseMatcher().with_message_ids(
+ [first_message_id, second_message_id]
+ ).with_messages_data([self._msg_body_template.format('0'), self._msg_body_template.format('1')])
+ )
+
+ def test_read_dont_stall(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False)
+ first_pack_size = 5
+ second_pack_size = 5
+ first_pack_ids = self._send_messages(
+ queue_url, message_count=first_pack_size, msg_body_template=self._msg_body_template
+ )
+ self._send_messages(queue_url, message_count=second_pack_size, msg_body_template=self._msg_body_template)
+ self._read_messages_and_assert(
queue_url, messages_count=10, visibility_timeout=1000, matcher=ReadResponseMatcher(
).with_n_or_more_messages(first_pack_size + 1).with_these_or_more_message_ids(first_pack_ids)
- )
-
- def test_multi_read_dont_stall(self):
- queue_url = self._create_queue_and_assert(self.queue_name)
- pack_size = 7
- total_packs = 3
- all_ids = set()
- read_ids = set()
- for _ in range(total_packs):
- all_ids.update(self._send_messages(
- queue_url, message_count=pack_size, msg_body_template=self._msg_body_template
- ))
- result = self._read_messages_and_assert(
+ )
+
+ def test_multi_read_dont_stall(self):
+ queue_url = self._create_queue_and_assert(self.queue_name)
+ pack_size = 7
+ total_packs = 3
+ all_ids = set()
+ read_ids = set()
+ for _ in range(total_packs):
+ all_ids.update(self._send_messages(
+ queue_url, message_count=pack_size, msg_body_template=self._msg_body_template
+ ))
+ result = self._read_messages_and_assert(
queue_url, messages_count=10, visibility_timeout=1000,
- matcher=ReadResponseMatcher().with_some_of_message_ids(all_ids - read_ids)
- )
- if result:
- read_ids.update(extract_message_ids(result))
-
- assert_that(
- len(read_ids), greater_than(pack_size),
- "Wrote {packs} packs of size {size}, but got only {total_read} messages after all read attempts".format(
- packs=total_packs, size=pack_size, total_read=len(read_ids)
- )
- )
-
- def test_visibility_timeout_works(self):
- total_msg_count = 10
+ matcher=ReadResponseMatcher().with_some_of_message_ids(all_ids - read_ids)
+ )
+ if result:
+ read_ids.update(extract_message_ids(result))
+
+ assert_that(
+ len(read_ids), greater_than(pack_size),
+ "Wrote {packs} packs of size {size}, but got only {total_read} messages after all read attempts".format(
+ packs=total_packs, size=pack_size, total_read=len(read_ids)
+ )
+ )
+
+ def test_visibility_timeout_works(self):
+ total_msg_count = 10
visibility_timeout = 5
before_read_time = time.time()
- self._create_queue_send_x_messages_read_y_messages(
+ self._create_queue_send_x_messages_read_y_messages(
self.queue_name, send_count=total_msg_count, read_count=1, visibility_timeout=visibility_timeout,
msg_body_template=self._msg_body_template, is_fifo=False
)
- msg_data = [self._msg_body_template.format(i) for i in range(total_msg_count)]
-
- already_read_id = extract_message_ids(self.read_result)[0]
+ msg_data = [self._msg_body_template.format(i) for i in range(total_msg_count)]
+
+ already_read_id = extract_message_ids(self.read_result)[0]
read_result = self._read_messages_and_assert(
self.queue_url, messages_count=5, visibility_timeout=visibility_timeout, wait_timeout=1,
- matcher=ReadResponseMatcher().with_some_of_message_ids(self.message_ids)
+ matcher=ReadResponseMatcher().with_some_of_message_ids(self.message_ids)
.with_n_messages(5)
)
@@ -396,25 +396,25 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
assert_that(read_ids, not_(has_item(already_read_id)))
time.sleep(visibility_timeout + 0.1)
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
self.queue_url, messages_count=10, visibility_timeout=1000,
- matcher=ReadResponseMatcher().with_message_ids(self.message_ids).with_messages_data(msg_data)
- )
-
- def test_visibility_timeout_expires_on_wait_timeout(self):
- queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False)
- message_ids = self._send_messages(
- queue_url, message_count=10, msg_body_template=self._msg_body_template
- )
- # noinspection PyTypeChecker
- self._read_messages_and_assert(
- queue_url, messages_count=10, matcher=ReadResponseMatcher().with_message_ids(message_ids),
- visibility_timeout=9
- )
- self._read_messages_and_assert(
+ matcher=ReadResponseMatcher().with_message_ids(self.message_ids).with_messages_data(msg_data)
+ )
+
+ def test_visibility_timeout_expires_on_wait_timeout(self):
+ queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False)
+ message_ids = self._send_messages(
+ queue_url, message_count=10, msg_body_template=self._msg_body_template
+ )
+ # noinspection PyTypeChecker
+ self._read_messages_and_assert(
+ queue_url, messages_count=10, matcher=ReadResponseMatcher().with_message_ids(message_ids),
+ visibility_timeout=9
+ )
+ self._read_messages_and_assert(
queue_url, messages_count=10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_some_of_message_ids(message_ids), wait_timeout=10,
- )
-
+ )
+
@pytest.mark.parametrize(**IS_FIFO_PARAMS)
def test_zero_visibility_timeout_works(self, is_fifo):
if is_fifo:
@@ -633,24 +633,24 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
raises(RuntimeError, pattern='InvalidParameterValue')
)
- def test_delete_message_works(self):
- self._create_queue_send_x_messages_read_y_messages(
+ def test_delete_message_works(self):
+ self._create_queue_send_x_messages_read_y_messages(
self.queue_name, send_count=10, read_count=2, visibility_timeout=0,
- msg_body_template=self._msg_body_template
+ msg_body_template=self._msg_body_template
)
handle = self.read_result[0]['ReceiptHandle']
- self.message_ids.remove(self.read_result[0]['MessageId'])
-
+ self.message_ids.remove(self.read_result[0]['MessageId'])
+
assert_that(
- self._sqs_api.delete_message(self.queue_url, handle), not_none()
+ self._sqs_api.delete_message(self.queue_url, handle), not_none()
)
# check double deletion
assert_that(
self._sqs_api.delete_message(self.queue_url, handle), not_none()
)
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
self.queue_url, messages_count=10, visibility_timeout=1000,
- matcher=ReadResponseMatcher().with_message_ids(self.message_ids)
+ matcher=ReadResponseMatcher().with_message_ids(self.message_ids)
)
counters = self._get_sqs_counters()
@@ -774,37 +774,37 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
else:
assert_that(messages_count_metric, equal_to(0))
- def test_can_read_new_written_data_on_visibility_timeout(self):
- visibility_timeout = 15
- self._create_queue_send_x_messages_read_y_messages(
- self.queue_name, send_count=7, read_count=4, visibility_timeout=visibility_timeout,
- msg_body_template=self._msg_body_template
+ def test_can_read_new_written_data_on_visibility_timeout(self):
+ visibility_timeout = 15
+ self._create_queue_send_x_messages_read_y_messages(
+ self.queue_name, send_count=7, read_count=4, visibility_timeout=visibility_timeout,
+ msg_body_template=self._msg_body_template
)
begin_time = time.time()
- first_pack_ids = self.message_ids
- second_pack_ids = self._send_messages(
- self.queue_url, message_count=3, msg_body_template=self._msg_body_template
- )
- message_ids = set(first_pack_ids) - set(extract_message_ids(self.read_result))
- message_ids.update(second_pack_ids)
+ first_pack_ids = self.message_ids
+ second_pack_ids = self._send_messages(
+ self.queue_url, message_count=3, msg_body_template=self._msg_body_template
+ )
+ message_ids = set(first_pack_ids) - set(extract_message_ids(self.read_result))
+ message_ids.update(second_pack_ids)
visibility_timeout_2 = 15
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
self.queue_url, messages_count=10, visibility_timeout=visibility_timeout_2,
matcher=ReadResponseMatcher().with_these_or_more_message_ids(message_ids).with_n_or_more_messages(6),
- )
+ )
remaining_time = visibility_timeout - (time.time() - begin_time) # for first pack read
time.sleep(max(remaining_time + 0.1, visibility_timeout_2 + 0.1, 0))
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
self.queue_url, messages_count=10, visibility_timeout=1000, matcher=ReadResponseMatcher().with_message_ids(
- first_pack_ids + second_pack_ids
- )
- )
+ first_pack_ids + second_pack_ids
+ )
+ )
- def test_partial_delete_works(self):
- self._create_queue_send_x_messages_read_y_messages(
- self.queue_name, send_count=10, read_count=5, visibility_timeout=5,
- msg_body_template=self._msg_body_template
+ def test_partial_delete_works(self):
+ self._create_queue_send_x_messages_read_y_messages(
+ self.queue_name, send_count=10, read_count=5, visibility_timeout=5,
+ msg_body_template=self._msg_body_template
)
handle = self.read_result[4]['ReceiptHandle'] # select the last read message to avoid race with visibility timeout while reading
self.message_ids.remove(self.read_result[4]['MessageId'])
@@ -813,18 +813,18 @@ class SqsGenericMessagingTest(KikimrSqsTestBase):
not_none()
)
time.sleep(6)
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
self.queue_url, messages_count=10, visibility_timeout=1000,
- matcher=ReadResponseMatcher().with_message_ids(self.message_ids)
+ matcher=ReadResponseMatcher().with_message_ids(self.message_ids)
)
-
+
def test_wrong_delete_fails(self):
self._create_queue_send_x_messages_read_y_messages(
self.queue_name, send_count=1, read_count=1, visibility_timeout=5,
msg_body_template=self._msg_body_template
)
handle = self.read_result[0]['ReceiptHandle']
-
+
def call_delete_invalid_handle():
self._sqs_api.delete_message(self.queue_url, handle + handle), # wrong handle
assert_that(
diff --git a/ydb/tests/functional/sqs/test_multinode_cluster.py b/ydb/tests/functional/sqs/test_multinode_cluster.py
index 6622e0ce19..047ae56198 100644
--- a/ydb/tests/functional/sqs/test_multinode_cluster.py
+++ b/ydb/tests/functional/sqs/test_multinode_cluster.py
@@ -4,7 +4,7 @@ import logging
import time
import threading
-import pytest
+import pytest
from hamcrest import assert_that, equal_to, not_none, raises, not_
from ydb.tests.library.common.types import Erasure
diff --git a/ydb/tests/functional/sqs/test_polling.py b/ydb/tests/functional/sqs/test_polling.py
index f8586dacca..dba0556e7d 100644
--- a/ydb/tests/functional/sqs/test_polling.py
+++ b/ydb/tests/functional/sqs/test_polling.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import pytest
+import pytest
from hamcrest import assert_that, equal_to
from sqs_matchers import ReadResponseMatcher
diff --git a/ydb/tests/functional/sqs/test_queue_attributes_validation.py b/ydb/tests/functional/sqs/test_queue_attributes_validation.py
index 8990056ef7..27ec79c9ee 100644
--- a/ydb/tests/functional/sqs/test_queue_attributes_validation.py
+++ b/ydb/tests/functional/sqs/test_queue_attributes_validation.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import pytest
+import pytest
from hamcrest import assert_that, equal_to
from sqs_test_base import KikimrSqsTestBase, IS_FIFO_PARAMS
diff --git a/ydb/tests/functional/sqs/test_queues_managing.py b/ydb/tests/functional/sqs/test_queues_managing.py
index 1e0d060d79..bcfd91ca67 100644
--- a/ydb/tests/functional/sqs/test_queues_managing.py
+++ b/ydb/tests/functional/sqs/test_queues_managing.py
@@ -3,8 +3,8 @@
import time
from collections import OrderedDict
-import pytest
-from hamcrest import assert_that, equal_to, greater_than, not_none, none, has_item, has_items, raises, empty, instance_of
+import pytest
+from hamcrest import assert_that, equal_to, greater_than, not_none, none, has_item, has_items, raises, empty, instance_of
from sqs_matchers import ReadResponseMatcher
@@ -21,10 +21,10 @@ class QueuesManagingTest(KikimrSqsTestBase):
return config_generator
@pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_create_queue(self, is_fifo):
+ def test_create_queue(self, is_fifo):
attributes = {}
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
attributes['ContentBasedDeduplication'] = 'true'
attributes['DelaySeconds'] = '506'
attributes['MaximumMessageSize'] = '10003'
@@ -32,11 +32,11 @@ class QueuesManagingTest(KikimrSqsTestBase):
attributes['ReceiveMessageWaitTimeSeconds'] = '11'
attributes['VisibilityTimeout'] = '42'
created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True)
- existing_queues = self._sqs_api.list_queues()
+ existing_queues = self._sqs_api.list_queues()
assert_that(
created_queue_url in existing_queues
)
- got_queue_url = self._sqs_api.get_queue_url(self.queue_name)
+ got_queue_url = self._sqs_api.get_queue_url(self.queue_name)
assert_that(
got_queue_url, equal_to(created_queue_url)
)
@@ -50,38 +50,38 @@ class QueuesManagingTest(KikimrSqsTestBase):
if is_fifo:
assert_that(created_attributes.get('ContentBasedDeduplication'), 'true')
- def test_create_fifo_queue_wo_postfix(self):
+ def test_create_fifo_queue_wo_postfix(self):
def call_create():
self.called = True
self._sqs_api.create_queue(self.queue_name, is_fifo=True)
- assert_that(
+ assert_that(
call_create,
raises(
RuntimeError,
pattern='failed with status 400.*\n.*FIFO queue should end with &quot;\\.fifo&quot;'
)
- )
-
- def test_create_queue_generates_event(self):
- pytest.skip("Outdated")
- self._create_queue_and_assert(self.queue_name, is_fifo=False)
+ )
+
+ def test_create_queue_generates_event(self):
+ pytest.skip("Outdated")
+ self._create_queue_and_assert(self.queue_name, is_fifo=False)
table_path = '{}/.Queues'.format(self.sqs_root)
- assert_that(self._get_table_lines_count(table_path), equal_to(1))
-
+ assert_that(self._get_table_lines_count(table_path), equal_to(1))
+
table_path = '{}/.Events'.format(self.sqs_root)
- assert_that(self._get_table_lines_count(table_path), equal_to(1))
-
- def test_remove_queue_generates_event(self):
- pytest.skip("Outdated")
- queue_url = self._create_queue_and_assert(self.queue_name)
+ assert_that(self._get_table_lines_count(table_path), equal_to(1))
+
+ def test_remove_queue_generates_event(self):
+ pytest.skip("Outdated")
+ queue_url = self._create_queue_and_assert(self.queue_name)
table_path = '{}/.Events'.format(self.sqs_root)
- lines_count = self._get_table_lines_count(table_path)
- assert_that(lines_count, greater_than(0))
-
- self._sqs_api.delete_queue(queue_url)
- assert_that(self._get_table_lines_count(table_path), greater_than(lines_count))
-
+ lines_count = self._get_table_lines_count(table_path)
+ assert_that(lines_count, greater_than(0))
+
+ self._sqs_api.delete_queue(queue_url)
+ assert_that(self._get_table_lines_count(table_path), greater_than(lines_count))
+
def test_create_queue_with_invalid_name(self):
def call_create():
self._sqs_api.create_queue('invalid_queue_name!')
@@ -95,11 +95,11 @@ class QueuesManagingTest(KikimrSqsTestBase):
)
@pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_delete_queue(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
- created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
- self._sqs_api.list_queues()
+ def test_delete_queue(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
+ created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo)
+ self._sqs_api.list_queues()
self._sqs_api.send_message(created_queue_url, 'body', group_id='group' if is_fifo else None, deduplication_id='123' if is_fifo else None)
send_message_labels = {
@@ -112,12 +112,12 @@ class QueuesManagingTest(KikimrSqsTestBase):
sends = self._get_counter_value(counters, send_message_labels)
assert_that(sends, equal_to(1))
- delete_result = self._sqs_api.delete_queue(created_queue_url)
+ delete_result = self._sqs_api.delete_queue(created_queue_url)
assert_that(
delete_result, not_none()
)
- existing_queues = self._sqs_api.list_queues()
+ existing_queues = self._sqs_api.list_queues()
assert_that(
created_queue_url not in existing_queues,
"Deleted queue appears in list_queues()"
@@ -238,16 +238,16 @@ class QueuesManagingTest(KikimrSqsTestBase):
check_purged_queue(created_queue_url2)
@pytest.mark.parametrize(**IS_FIFO_PARAMS)
- def test_delete_and_create_queue(self, is_fifo):
- if is_fifo:
- self.queue_name = self.queue_name + '.fifo'
+ def test_delete_and_create_queue(self, is_fifo):
+ if is_fifo:
+ self.queue_name = self.queue_name + '.fifo'
created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True)
self.seq_no += 1
self._send_message_and_assert(created_queue_url, self._msg_body_template.format(1), seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None)
- delete_result = self._sqs_api.delete_queue(created_queue_url)
- assert_that(
- delete_result, not_none()
- )
+ delete_result = self._sqs_api.delete_queue(created_queue_url)
+ assert_that(
+ delete_result, not_none()
+ )
created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=is_fifo, use_http=True)
master_is_updated = False
@@ -265,9 +265,9 @@ class QueuesManagingTest(KikimrSqsTestBase):
assert_that(master_is_updated)
self.seq_no += 1
msg_id = self._send_message_and_assert(created_queue_url, self._msg_body_template.format(2), seq_no=self.seq_no if is_fifo else None, group_id='group' if is_fifo else None)
- self._read_messages_and_assert(
+ self._read_messages_and_assert(
created_queue_url, 10, ReadResponseMatcher().with_message_ids([msg_id, ])
- )
+ )
def test_ya_count_queues(self):
assert_that(self._sqs_api.private_count_queues(), equal_to('0'))
@@ -336,5 +336,5 @@ class TestQueuesManagingWithTenant(get_test_with_sqs_tenant_installation(QueuesM
pass
-class TestQueuesManagingWithPathTestQueuesManagingWithPath(get_test_with_sqs_installation_by_path(QueuesManagingTest)):
+class TestQueuesManagingWithPathTestQueuesManagingWithPath(get_test_with_sqs_installation_by_path(QueuesManagingTest)):
pass
diff --git a/ydb/tests/functional/sqs/test_recompiles_requests.py b/ydb/tests/functional/sqs/test_recompiles_requests.py
index 8053131116..367003323f 100644
--- a/ydb/tests/functional/sqs/test_recompiles_requests.py
+++ b/ydb/tests/functional/sqs/test_recompiles_requests.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import pytest
+import pytest
from hamcrest import assert_that, not_none
from ydb.tests.library.common.types import Erasure
diff --git a/ydb/tests/functional/sqs/ya.make b/ydb/tests/functional/sqs/ya.make
index c9fb9f0623..bbb0a57bb8 100644
--- a/ydb/tests/functional/sqs/ya.make
+++ b/ydb/tests/functional/sqs/ya.make
@@ -27,7 +27,7 @@ TEST_SRCS(
)
IF (SANITIZER_TYPE)
- TIMEOUT(2400)
+ TIMEOUT(2400)
SIZE(LARGE)
TAG(ya:fat)
REQUIREMENTS(
diff --git a/ydb/tests/library/common/composite_assert.py b/ydb/tests/library/common/composite_assert.py
index 4c8c1075c6..38087494cb 100644
--- a/ydb/tests/library/common/composite_assert.py
+++ b/ydb/tests/library/common/composite_assert.py
@@ -44,28 +44,28 @@ class CompositeAssert(object):
return False
-# ToDo: Refactor to use CompositeAssert instead of this.
-class CompositeCheckResult(object):
- def __init__(self):
- self.__result = True
- self.__reason = []
-
- @property
- def result(self):
- return self.__result
-
- @property
- def description(self):
- return '\n'.join(self.__reason)
-
- def check(self, condition, description=""):
- if not condition:
- self.set_failed(description)
-
- def set_failed(self, description=""):
- self.__result = False
- if description:
- self.__reason.append(description)
-
- def __nonzero__(self):
- return self.__result
+# ToDo: Refactor to use CompositeAssert instead of this.
+class CompositeCheckResult(object):
+ def __init__(self):
+ self.__result = True
+ self.__reason = []
+
+ @property
+ def result(self):
+ return self.__result
+
+ @property
+ def description(self):
+ return '\n'.join(self.__reason)
+
+ def check(self, condition, description=""):
+ if not condition:
+ self.set_failed(description)
+
+ def set_failed(self, description=""):
+ self.__result = False
+ if description:
+ self.__reason.append(description)
+
+ def __nonzero__(self):
+ return self.__result
diff --git a/ydb/tests/library/common/generators.py b/ydb/tests/library/common/generators.py
index ba99d3c6fc..8261535a30 100644
--- a/ydb/tests/library/common/generators.py
+++ b/ydb/tests/library/common/generators.py
@@ -1,49 +1,49 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import random
-import string
-
-__author__ = 'asatarin@yandex-team.ru'
-
-
-__hex_digits = '0123456789ABCDEF'
-
-
-def int_between(lo, hi):
- def closure():
- while True:
- yield random.randint(lo, hi)
- return closure
-
-
-def one_of(list_of_values):
- def closure():
- while True:
- yield random.choice(list_of_values)
- return closure
-
-
-def float_in(lo, hi):
- def closure():
- while True:
- yield random.uniform(lo, hi)
- return closure
-
-
-def float_as_hex(byte_length=4):
- def closure():
- while True:
- yield 'x"' + ''.join([random.choice(__hex_digits) for _ in range(byte_length * 2)]) + '"'
- return closure
-
-
-# TODO use string.printable for character generation or even wider range (e.g. UTF-8)
-def string_with_length(length):
- def closure():
- while True:
- yield ''.join([random.choice(string.ascii_letters) for _ in range(length)])
- return closure
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import random
+import string
+
+__author__ = 'asatarin@yandex-team.ru'
+
+
+__hex_digits = '0123456789ABCDEF'
+
+
+def int_between(lo, hi):
+ def closure():
+ while True:
+ yield random.randint(lo, hi)
+ return closure
+
+
+def one_of(list_of_values):
+ def closure():
+ while True:
+ yield random.choice(list_of_values)
+ return closure
+
+
+def float_in(lo, hi):
+ def closure():
+ while True:
+ yield random.uniform(lo, hi)
+ return closure
+
+
+def float_as_hex(byte_length=4):
+ def closure():
+ while True:
+ yield 'x"' + ''.join([random.choice(__hex_digits) for _ in range(byte_length * 2)]) + '"'
+ return closure
+
+
+# TODO use string.printable for character generation or even wider range (e.g. UTF-8)
+def string_with_length(length):
+ def closure():
+ while True:
+ yield ''.join([random.choice(string.ascii_letters) for _ in range(length)])
+ return closure
def actor_id():
diff --git a/ydb/tests/library/common/protobuf_ss.py b/ydb/tests/library/common/protobuf_ss.py
index 65045bc8c3..90b0638eb4 100644
--- a/ydb/tests/library/common/protobuf_ss.py
+++ b/ydb/tests/library/common/protobuf_ss.py
@@ -9,9 +9,9 @@ from ydb.core.protos import flat_scheme_op_pb2
from ydb.tests.library.common.protobuf import AbstractProtobufBuilder, build_protobuf_if_necessary
-DEFAULT_SIZE_TO_SPLIT = 10 ** 6
-
-
+DEFAULT_SIZE_TO_SPLIT = 10 ** 6
+
+
class TPartitionConfig(AbstractProtobufBuilder):
"""
See /arcadia/ydb/core/protos/flat_scheme_op_pb2.proto
@@ -19,7 +19,7 @@ class TPartitionConfig(AbstractProtobufBuilder):
def __init__(self):
super(TPartitionConfig, self).__init__(flat_scheme_op_pb2.TPartitionConfig())
- self.with_partitioning_policy(DEFAULT_SIZE_TO_SPLIT)
+ self.with_partitioning_policy(DEFAULT_SIZE_TO_SPLIT)
def __ensure_has_compaction_policy(self):
if not self.protobuf.HasField('CompactionPolicy'):
diff --git a/ydb/tests/library/common/yatest_common.py b/ydb/tests/library/common/yatest_common.py
index ba7c3b9ce2..0b83695155 100644
--- a/ydb/tests/library/common/yatest_common.py
+++ b/ydb/tests/library/common/yatest_common.py
@@ -30,9 +30,9 @@ binary_path = wrap(ya_common.binary_path, lambda x: x)
output_path = wrap(ya_common.output_path, lambda x: x)
work_path = wrap(ya_common.work_path, lambda x: x)
-get_param = wrap(ya_common.get_param, lambda x, y=None: y)
+get_param = wrap(ya_common.get_param, lambda x, y=None: y)
get_param_dict_copy = wrap(ya_common.get_param_dict_copy, lambda: dict())
-
+
def get_bool_param(key, default):
val = get_param(key, default)
@@ -67,4 +67,4 @@ def plain_or_under_sanitizer(plain, sanitized):
:return: plain if no sanitizer enabled or sanitized otherwise
"""
- return plain if not context.sanitize else sanitized
+ return plain if not context.sanitize else sanitized
diff --git a/ydb/tests/library/harness/daemon.py b/ydb/tests/library/harness/daemon.py
index 5c0055bf3c..edabe07262 100644
--- a/ydb/tests/library/harness/daemon.py
+++ b/ydb/tests/library/harness/daemon.py
@@ -178,7 +178,7 @@ class Daemon(object):
if not is_killed:
exit_code = self.__daemon.exit_code
self.__check_before_fail()
-
+
if exit_code not in self._acceptable_exit_codes:
raise DaemonError(
"Bad exit_code.",
diff --git a/ydb/tests/library/harness/kikimr_client.py b/ydb/tests/library/harness/kikimr_client.py
index c89add5a68..d74383ae23 100644
--- a/ydb/tests/library/harness/kikimr_client.py
+++ b/ydb/tests/library/harness/kikimr_client.py
@@ -91,9 +91,9 @@ class KiKiMRMessageBusClient(object):
time.sleep(self.__retry_sleep_seconds)
- def close(self):
+ def close(self):
self._channel.close()
-
+
def update_self_heal(self, enable, domain=1):
request = msgbus.TBlobStorageConfigRequest()
request.Domain = domain
@@ -392,5 +392,5 @@ class KiKiMRMessageBusClient(object):
request.Alive = True
return self.invoke(request, 'TabletStateRequest')
- def __del__(self):
- self.close()
+ def __del__(self):
+ self.close()
diff --git a/ydb/tests/library/harness/kikimr_cluster.py b/ydb/tests/library/harness/kikimr_cluster.py
index 253e415d69..942e1588ca 100644
--- a/ydb/tests/library/harness/kikimr_cluster.py
+++ b/ydb/tests/library/harness/kikimr_cluster.py
@@ -1,13 +1,13 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
-import itertools
-import logging
+import itertools
+import logging
import subprocess
import time
import pprint
from concurrent import futures
-
+
import ydb.tests.library.common.yatest_common as yatest_common
from . import param_constants
diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py
index 2ffa0d6b3a..57c282b9f4 100644
--- a/ydb/tests/library/harness/kikimr_config.py
+++ b/ydb/tests/library/harness/kikimr_config.py
@@ -368,7 +368,7 @@ class KikimrConfigGenerator(object):
{"vdisk_locations": [{"node_id": node_id, "pdisk_id": pdisk_id, "pdisk_guid": pdisk_id, 'vdisk_slot_id': 0}]}
)
- def __build(self):
+ def __build(self):
datacenter_id_generator = itertools.cycle(self._dcs)
self.yaml_config["blob_storage_config"] = {}
self.yaml_config["blob_storage_config"]["service_set"] = {}
diff --git a/ydb/tests/library/harness/kikimr_http_client.py b/ydb/tests/library/harness/kikimr_http_client.py
index fe023bedb3..68bbfe7375 100644
--- a/ydb/tests/library/harness/kikimr_http_client.py
+++ b/ydb/tests/library/harness/kikimr_http_client.py
@@ -55,40 +55,40 @@ class HiveClient(object):
url_lst.append('channel=' + ','.join(map(str, channels)))
self.__get('&'.join(url_lst), timeout=TIMEOUT)
- def kick_tablets_from_node(self, node_id):
+ def kick_tablets_from_node(self, node_id):
self.__get(
- self.__url + "&page=KickNode&node={node}".format(node=node_id),
- timeout=TIMEOUT
- )
+ self.__url + "&page=KickNode&node={node}".format(node=node_id),
+ timeout=TIMEOUT
+ )
- def block_node(self, node_id, block=True):
- block = '1' if block else '0'
+ def block_node(self, node_id, block=True):
+ block = '1' if block else '0'
self.__get(
- self.__url + "&page=SetDown&node={node}&down={down}".format(node=node_id, down=block),
- timeout=TIMEOUT
- )
-
- def unblock_node(self, node_id):
+ self.__url + "&page=SetDown&node={node}&down={down}".format(node=node_id, down=block),
+ timeout=TIMEOUT
+ )
+
+ def unblock_node(self, node_id):
self.block_node(node_id, block=False)
-
- def change_tablet_weight(self, tablet_id, tablet_weight):
+
+ def change_tablet_weight(self, tablet_id, tablet_weight):
self.__get(
- self.__url + "&page=UpdateResources&tablet={tablet}&kv={size}".format(
- tablet=tablet_id,
- size=tablet_weight
- ),
- timeout=TIMEOUT
- )
-
- def change_tablet_cpu_usage(self, tablet_id, cpu_usage):
+ self.__url + "&page=UpdateResources&tablet={tablet}&kv={size}".format(
+ tablet=tablet_id,
+ size=tablet_weight
+ ),
+ timeout=TIMEOUT
+ )
+
+ def change_tablet_cpu_usage(self, tablet_id, cpu_usage):
self.__get(
- self.__url + "&page=UpdateResources&tablet={tablet}&cpu={size}".format(
- tablet=tablet_id,
- size=cpu_usage
- ),
- timeout=TIMEOUT
- )
-
+ self.__url + "&page=UpdateResources&tablet={tablet}&cpu={size}".format(
+ tablet=tablet_id,
+ size=cpu_usage
+ ),
+ timeout=TIMEOUT
+ )
+
def set_min_scatter_to_balance(self, min_scatter_to_balance=101):
"""
min_scatter_to_balance=101 -- effectively disables auto rebalancing
@@ -97,7 +97,7 @@ class HiveClient(object):
self.__url + "&page=Settings&minScatterToBalance={min_scatter_to_balance}".format(
min_scatter_to_balance=min_scatter_to_balance), timeout=TIMEOUT
)
-
+
def set_max_scheduled_tablets(self, max_scheduled_tablets=10):
self.__get(
self.__url + "&page=Settings&maxScheduledTablets={max_scheduled_tablets}".format(
@@ -158,17 +158,17 @@ class SwaggerClient(object):
"/json/pdiskinfo", node_id=str(node_id), enums='true', timeout=self.__timeout
)
- def vdisk_info(self):
- return self.__http_get_and_parse_json(
- "/json/vdiskinfo", timeout=self.__timeout
- )
-
- def tablet_info(self, tablet_type=None):
- if tablet_type is not None:
- return self.__http_get_and_parse_json(
- "/json/tabletinfo", timeout=self.__timeout, filter='Type={}'.format(tablet_type)
- )
- else:
- return self.__http_get_and_parse_json(
- "/json/tabletinfo", timeout=self.__timeout
- )
+ def vdisk_info(self):
+ return self.__http_get_and_parse_json(
+ "/json/vdiskinfo", timeout=self.__timeout
+ )
+
+ def tablet_info(self, tablet_type=None):
+ if tablet_type is not None:
+ return self.__http_get_and_parse_json(
+ "/json/tabletinfo", timeout=self.__timeout, filter='Type={}'.format(tablet_type)
+ )
+ else:
+ return self.__http_get_and_parse_json(
+ "/json/tabletinfo", timeout=self.__timeout
+ )
diff --git a/ydb/tests/library/harness/kikimr_runner.py b/ydb/tests/library/harness/kikimr_runner.py
index 25803842c3..251f485b5d 100644
--- a/ydb/tests/library/harness/kikimr_runner.py
+++ b/ydb/tests/library/harness/kikimr_runner.py
@@ -30,7 +30,7 @@ def get_unique_path_for_current_test(output_path, sub_folder):
return os.path.join(output_path, test_name, sub_folder)
-
+
def ensure_path_exists(path):
if not os.path.isdir(path):
os.makedirs(path)
@@ -119,20 +119,20 @@ class KiKiMRNode(daemon.Daemon, kikimr_node_interface.NodeInterface):
if self.__configurator.suppress_version_check:
command.append("--suppress-version-check")
- if self.__node_broker_port is not None:
+ if self.__node_broker_port is not None:
command.append("--node-broker=%s%s:%d" % (
"grpcs://" if self.__configurator.grpc_ssl_enable else "",
self.host,
self.__node_broker_port))
else:
- command.append("--node=%d" % self.node_id)
+ command.append("--node=%d" % self.node_id)
if self.__configurator.grpc_ssl_enable:
command.append(
"--ca=%s" % self.__configurator.grpc_tls_ca_path
)
- if self.__role == 'slot':
+ if self.__role == 'slot':
command.append(
"--tenant=%s" % self._tenant_affiliation
)
@@ -165,12 +165,12 @@ class KiKiMRNode(daemon.Daemon, kikimr_node_interface.NodeInterface):
logger.info("Final command: %s", ' '.join(command).replace(self.__config_path, '$CFG_DIR_PATH'))
return command
- def stop(self):
+ def stop(self):
try:
super(KiKiMRNode, self).stop()
finally:
logger.info("Stopped node %s", self)
-
+
def kill(self):
try:
super(KiKiMRNode, self).kill()
@@ -431,7 +431,7 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface):
self.nodes[node_id].format_pdisk(**pdisk)
def __add_bs_box(self):
- request = bs.TConfigRequest()
+ request = bs.TConfigRequest()
for node_id in self.__configurator.all_node_ids():
cmd = request.Command.add()
@@ -483,7 +483,7 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface):
def add_storage_pool(self, name=None, kind="rot", pdisk_user_kind=0, erasure=None):
if erasure is None:
erasure = self.__configurator.static_erasure
- request = bs.TConfigRequest()
+ request = bs.TConfigRequest()
cmd = request.Command.add()
cmd.DefineStoragePool.BoxId = 1
@@ -505,16 +505,16 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface):
def __wait_for_bs_controller_to_start(self):
monitors = [node.monitor for node in self.nodes.values()]
-
- def predicate():
- return blobstorage_controller_has_started_on_some_node(monitors)
-
+
+ def predicate():
+ return blobstorage_controller_has_started_on_some_node(monitors)
+
timeout_seconds = yatest_common.plain_or_under_sanitizer(120, 240)
bs_controller_started = wait_for(
predicate=predicate, timeout_seconds=timeout_seconds, step_seconds=1.0, multiply=1.3
)
assert bs_controller_started
-
+
class KikimrExternalNode(daemon.ExternalNodeDaemon, kikimr_node_interface.NodeInterface):
def __init__(
diff --git a/ydb/tests/library/matchers/response.py b/ydb/tests/library/matchers/response.py
index 2fe460597d..d492116b4b 100644
--- a/ydb/tests/library/matchers/response.py
+++ b/ydb/tests/library/matchers/response.py
@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import six
-from hamcrest import all_of, has_property, has_properties
+from hamcrest import all_of, has_property, has_properties
from hamcrest.core.base_matcher import BaseMatcher
import ydb.core.protos.msgbus_kv_pb2 as msgbus_kv
@@ -26,56 +26,56 @@ def to_bytes(v):
class KeyValueResponseProtobufMatcher(BaseMatcher):
- def __init__(self, status=MessageBusStatus.MSTATUS_OK, do_describe=True):
+ def __init__(self, status=MessageBusStatus.MSTATUS_OK, do_describe=True):
self.__read_range_matchers = []
self.__read_keys_matchers = []
- self.__write_matchers = []
- self.__rename_matchers = []
- self.__delete_matcher = None
+ self.__write_matchers = []
+ self.__rename_matchers = []
+ self.__delete_matcher = None
self.__copy_range_matchers = []
self.__concat_keys_result_matchers = []
self.__inc_gen_result_matcher = None
self.__storage_channel_result_matchers = []
- self.__do_describe = do_describe
+ self.__do_describe = do_describe
self.__describe_actual_proto_fields = set()
self.__status = status
def no_describe(self, describe_fields=()):
- self.__do_describe = False
+ self.__do_describe = False
self.__describe_actual_proto_fields = set(describe_fields)
- return self
-
+ return self
+
def describe_to(self, description):
description.append_text('Valid response with Status: %i' % self.__status)
- if self.__do_describe:
- if self.__inc_gen_result_matcher is not None:
- description.append_text(' and IncrementGenerationResult = \n')
- self.__inc_gen_result_matcher.describe_to(description)
- if self.__write_matchers:
- description.append_text(' and WriteResult = \n')
- contains(*self.__write_matchers).describe_to(description)
- if self.__delete_matcher is not None:
- description.append_text(' and DeleteRangeResult = \n')
- self.__delete_matcher.describe_to(description)
- if self.__read_keys_matchers:
- description.append_text(' and ReadResult = \n')
- contains(*self.__read_keys_matchers).describe_to(description)
- if self.__read_range_matchers:
- description.append_text(' and ReadRangeResult = \n')
- contains(*self.__read_range_matchers).describe_to(description)
- if self.__copy_range_matchers:
- description.append_text(' and CopyRangeResult = \n')
- contains(*self.__copy_range_matchers).describe_to(description)
- if self.__concat_keys_result_matchers:
- description.append_text(' and ConcatResult = \n')
- contains(*self.__concat_keys_result_matchers).describe_to(description)
- else:
- description.append_text("(Verbose mismatch description was disabled for this test)")
+ if self.__do_describe:
+ if self.__inc_gen_result_matcher is not None:
+ description.append_text(' and IncrementGenerationResult = \n')
+ self.__inc_gen_result_matcher.describe_to(description)
+ if self.__write_matchers:
+ description.append_text(' and WriteResult = \n')
+ contains(*self.__write_matchers).describe_to(description)
+ if self.__delete_matcher is not None:
+ description.append_text(' and DeleteRangeResult = \n')
+ self.__delete_matcher.describe_to(description)
+ if self.__read_keys_matchers:
+ description.append_text(' and ReadResult = \n')
+ contains(*self.__read_keys_matchers).describe_to(description)
+ if self.__read_range_matchers:
+ description.append_text(' and ReadRangeResult = \n')
+ contains(*self.__read_range_matchers).describe_to(description)
+ if self.__copy_range_matchers:
+ description.append_text(' and CopyRangeResult = \n')
+ contains(*self.__copy_range_matchers).describe_to(description)
+ if self.__concat_keys_result_matchers:
+ description.append_text(' and ConcatResult = \n')
+ contains(*self.__concat_keys_result_matchers).describe_to(description)
+ else:
+ description.append_text("(Verbose mismatch description was disabled for this test)")
def describe_mismatch(self, actual_protobuf, mismatch_description):
if self.__do_describe and not self.__describe_actual_proto_fields:
actual_protobuf_str = str(actual_protobuf)
- else:
+ else:
lst = []
field_names = [f.name for f in actual_protobuf.DESCRIPTOR.fields]
for f in field_names:
@@ -89,7 +89,7 @@ class KeyValueResponseProtobufMatcher(BaseMatcher):
lst.append("{f} = {v}".format(f=f, v=value))
lst.append('\n')
actual_protobuf_str = '\n'.join(lst)
-
+
mismatch_description.append_text("Actual protobuf = \n")
mismatch_description.append_text("=" * 60 + "\n")
mismatch_description.append_text(actual_protobuf_str)
@@ -99,45 +99,45 @@ class KeyValueResponseProtobufMatcher(BaseMatcher):
if not hasattr(actual_protobuf, 'Status') or actual_protobuf.Status != self.__status:
return False
- if self.__inc_gen_result_matcher is not None\
- and not self.__inc_gen_result_matcher.matches(actual_protobuf.IncrementGenerationResult):
+ if self.__inc_gen_result_matcher is not None\
+ and not self.__inc_gen_result_matcher.matches(actual_protobuf.IncrementGenerationResult):
return False
if (
- (self.__read_range_matchers and
- not contains(*self.__read_range_matchers).matches(actual_protobuf.ReadRangeResult)
- ) or
- (self.__rename_matchers and
- not contains(*self.__rename_matchers).matches(actual_protobuf.RenameResult)
- ) or
- (self.__write_matchers and
- not contains(*self.__write_matchers).matches(actual_protobuf.WriteResult)
- ) or
- (self.__delete_matcher is not None and
- not self.__delete_matcher.matches(actual_protobuf.DeleteRangeResult)
- ) or
- (self.__copy_range_matchers and
- not contains(*self.__copy_range_matchers).matches(actual_protobuf.CopyRangeResult)
- ) or
- (self.__concat_keys_result_matchers and
- not contains(*self.__concat_keys_result_matchers).matches(actual_protobuf.ConcatResult)
- ) or
- (self.__read_keys_matchers and
- not contains(*self.__read_keys_matchers).matches(actual_protobuf.ReadResult)
+ (self.__read_range_matchers and
+ not contains(*self.__read_range_matchers).matches(actual_protobuf.ReadRangeResult)
+ ) or
+ (self.__rename_matchers and
+ not contains(*self.__rename_matchers).matches(actual_protobuf.RenameResult)
+ ) or
+ (self.__write_matchers and
+ not contains(*self.__write_matchers).matches(actual_protobuf.WriteResult)
+ ) or
+ (self.__delete_matcher is not None and
+ not self.__delete_matcher.matches(actual_protobuf.DeleteRangeResult)
+ ) or
+ (self.__copy_range_matchers and
+ not contains(*self.__copy_range_matchers).matches(actual_protobuf.CopyRangeResult)
+ ) or
+ (self.__concat_keys_result_matchers and
+ not contains(*self.__concat_keys_result_matchers).matches(actual_protobuf.ConcatResult)
+ ) or
+ (self.__read_keys_matchers and
+ not contains(*self.__read_keys_matchers).matches(actual_protobuf.ReadResult)
) or
(self.__storage_channel_result_matchers and
not contains(*self.__storage_channel_result_matchers).matches(actual_protobuf.GetStatusResult)
- )
+ )
):
return False
return True
- def read_range(self, key_value_pairs, status=EReplyStatus.OK):
+ def read_range(self, key_value_pairs, status=EReplyStatus.OK):
key_value_pair_matchers = tuple(kv_pair.protobuf_matcher() for kv_pair in key_value_pairs)
self.__read_range_matchers.append(
has_properties(
- Status=status,
+ Status=status,
Pair=contains(*tuple(
key_value_pair_matchers
))
@@ -145,37 +145,37 @@ class KeyValueResponseProtobufMatcher(BaseMatcher):
)
return self
- def read_key(self, value, status=EReplyStatus.OK):
+ def read_key(self, value, status=EReplyStatus.OK):
self.__read_keys_matchers.append(
has_properties(
- Status=status,
+ Status=status,
Value=to_bytes(value)
)
)
return self
def write(self, num_of_writes=1):
- self.__write_matchers.extend(
+ self.__write_matchers.extend(
has_properties(Status=int(EReplyStatus.OK), StatusFlags=int(TStorageStatusFlags.StatusIsValid))
for _ in range(num_of_writes)
- )
- return self
-
+ )
+ return self
+
def add_write_result_with_status(self, status, flags):
self.__write_matchers.append(
has_properties(Status=int(status), StatusFlags=int(flags))
)
return self
- def rename(self, num_of_renames=1):
- self.__rename_matchers.extend(
- has_property('Status', int(EReplyStatus.OK)) for _ in range(num_of_renames)
- )
- return self
-
- def delete(self, num_of_keys=1):
- self.__delete_matcher = contains(*tuple(
- has_property('Status', EReplyStatus.OK) for _ in range(num_of_keys)
+ def rename(self, num_of_renames=1):
+ self.__rename_matchers.extend(
+ has_property('Status', int(EReplyStatus.OK)) for _ in range(num_of_renames)
+ )
+ return self
+
+ def delete(self, num_of_keys=1):
+ self.__delete_matcher = contains(*tuple(
+ has_property('Status', EReplyStatus.OK) for _ in range(num_of_keys)
))
return self
@@ -236,10 +236,10 @@ def is_response_with_status_and_fields(status, **kwargs):
)
-def is_valid_read_range_response(key_value_pairs,
- main_status=MessageBusStatus.MSTATUS_OK, result_status=EReplyStatus.OK):
- return KeyValueResponseProtobufMatcher(main_status).read_range(key_value_pairs, status=result_status)
+def is_valid_read_range_response(key_value_pairs,
+ main_status=MessageBusStatus.MSTATUS_OK, result_status=EReplyStatus.OK):
+ return KeyValueResponseProtobufMatcher(main_status).read_range(key_value_pairs, status=result_status)
-def is_valid_keyvalue_protobuf_response():
+def is_valid_keyvalue_protobuf_response():
return KeyValueResponseProtobufMatcher()
diff --git a/ydb/tests/library/matchers/tablets.py b/ydb/tests/library/matchers/tablets.py
index 925c1f8200..c73b8e4297 100644
--- a/ydb/tests/library/matchers/tablets.py
+++ b/ydb/tests/library/matchers/tablets.py
@@ -1,106 +1,106 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-from hamcrest import has_property, all_of, has_length, has_items, anything
-from hamcrest.core.base_matcher import BaseMatcher
-from hamcrest.core.helpers.wrap_matcher import wrap_matcher
+from hamcrest import has_property, all_of, has_length, has_items, anything
+from hamcrest.core.base_matcher import BaseMatcher
+from hamcrest.core.helpers.wrap_matcher import wrap_matcher
from ydb.tests.library.common.msgbus_types import EMessageStatus, MessageBusStatus
def all_tablets_are_created(tablets_to_create):
return all_of(
- has_property('Status', MessageBusStatus.MSTATUS_OK),
+ has_property('Status', MessageBusStatus.MSTATUS_OK),
has_property(
- 'CreateTabletResult',
+ 'CreateTabletResult',
all_of(
has_length(len(tablets_to_create)),
- has_items(has_property('Status', EMessageStatus.MESSAGE_OK))
+ has_items(has_property('Status', EMessageStatus.MESSAGE_OK))
)
)
)
-
-
-def is_balanced_list(lst, exact_values):
- _min, _max = min(lst), max(lst)
- if exact_values is not None:
- return _min == _max and wrap_matcher(exact_values).matches(_min)
- else:
- return _max - _min <= 1
-
-
-class ExpectedTablets(object):
- def __init__(self, tablets_count=anything(), tablet_ids=()):
- self.tablets_count = wrap_matcher(tablets_count)
- self.tablet_ids = list(tablet_ids)
-
-
-class TabletsBalanceMatcher(BaseMatcher):
- def __init__(self):
- super(TabletsBalanceMatcher, self).__init__()
- self.__all_balanced = False
- self.__special_nodes = {}
- self.__exact_count_on_all = None
-
- def all_balanced(self):
- self.__all_balanced = True
- return self
-
- def with_exact_count_tablets_on_all(self, exact_count):
- self.__exact_count_on_all = exact_count
- return self
-
- def with_x_tablets_on_node(self, node_id, num_tablets):
- self.__special_nodes.setdefault(node_id, ExpectedTablets())
- self.__special_nodes[node_id].tablets_count = num_tablets
- return self
-
- def with_exact_tablet_id_on_node(self, node_id, tablet_id):
- self.__special_nodes.setdefault(node_id, ExpectedTablets())
- self.__special_nodes[node_id].tablet_ids.append(tablet_id)
- return self
-
- def _matches(self, actual_tablets_per_node):
- not_special_nodes = actual_tablets_per_node.keys()
-
- for node_id, expected_tablets in self.__special_nodes.items():
- if node_id not in actual_tablets_per_node:
- return False
- if not expected_tablets.tablets_count.matches(len(actual_tablets_per_node[node_id])):
- return False
-
- for tablet_id in expected_tablets.tablet_ids:
- if tablet_id not in actual_tablets_per_node[node_id]:
- return False
- not_special_nodes.remove(node_id)
-
- tablets_by_node = [len(actual_tablets_per_node[i]) for i in not_special_nodes]
- if self.__all_balanced:
- return is_balanced_list(
- tablets_by_node,
- self.__exact_count_on_all
- )
- elif self.__exact_count_on_all is not None:
- return self.__exact_count_on_all.matches(
- max(tablets_by_node)
- ) and self.__exact_count_on_all.matches(
- min(tablets_by_node)
- )
-
- def describe_to(self, description):
- if self.__all_balanced:
- description.append_text('Equally spread tablets across all nodes\n')
- if self.__exact_count_on_all is not None:
- description.append_text('With each node having \n')
- self.__exact_count_on_all.describe_to(description)
- for node_id, expected_tablets in self.__special_nodes.items():
- description.append_text('Except node %s which is expected to have\n' % str(node_id))
- description.append_text('\t num tablets: ')
- expected_tablets.tablets_count.describe_to(description)
- description.append_text('.')
- for tablet_id in expected_tablets.tablet_ids:
- description.append_text('\t tablet with id %s on it' % str(tablet_id))
-
-
-def are_equally_spread_tablets():
- return TabletsBalanceMatcher().all_balanced()
+
+
+def is_balanced_list(lst, exact_values):
+ _min, _max = min(lst), max(lst)
+ if exact_values is not None:
+ return _min == _max and wrap_matcher(exact_values).matches(_min)
+ else:
+ return _max - _min <= 1
+
+
+class ExpectedTablets(object):
+ def __init__(self, tablets_count=anything(), tablet_ids=()):
+ self.tablets_count = wrap_matcher(tablets_count)
+ self.tablet_ids = list(tablet_ids)
+
+
+class TabletsBalanceMatcher(BaseMatcher):
+ def __init__(self):
+ super(TabletsBalanceMatcher, self).__init__()
+ self.__all_balanced = False
+ self.__special_nodes = {}
+ self.__exact_count_on_all = None
+
+ def all_balanced(self):
+ self.__all_balanced = True
+ return self
+
+ def with_exact_count_tablets_on_all(self, exact_count):
+ self.__exact_count_on_all = exact_count
+ return self
+
+ def with_x_tablets_on_node(self, node_id, num_tablets):
+ self.__special_nodes.setdefault(node_id, ExpectedTablets())
+ self.__special_nodes[node_id].tablets_count = num_tablets
+ return self
+
+ def with_exact_tablet_id_on_node(self, node_id, tablet_id):
+ self.__special_nodes.setdefault(node_id, ExpectedTablets())
+ self.__special_nodes[node_id].tablet_ids.append(tablet_id)
+ return self
+
+ def _matches(self, actual_tablets_per_node):
+ not_special_nodes = actual_tablets_per_node.keys()
+
+ for node_id, expected_tablets in self.__special_nodes.items():
+ if node_id not in actual_tablets_per_node:
+ return False
+ if not expected_tablets.tablets_count.matches(len(actual_tablets_per_node[node_id])):
+ return False
+
+ for tablet_id in expected_tablets.tablet_ids:
+ if tablet_id not in actual_tablets_per_node[node_id]:
+ return False
+ not_special_nodes.remove(node_id)
+
+ tablets_by_node = [len(actual_tablets_per_node[i]) for i in not_special_nodes]
+ if self.__all_balanced:
+ return is_balanced_list(
+ tablets_by_node,
+ self.__exact_count_on_all
+ )
+ elif self.__exact_count_on_all is not None:
+ return self.__exact_count_on_all.matches(
+ max(tablets_by_node)
+ ) and self.__exact_count_on_all.matches(
+ min(tablets_by_node)
+ )
+
+ def describe_to(self, description):
+ if self.__all_balanced:
+ description.append_text('Equally spread tablets across all nodes\n')
+ if self.__exact_count_on_all is not None:
+ description.append_text('With each node having \n')
+ self.__exact_count_on_all.describe_to(description)
+ for node_id, expected_tablets in self.__special_nodes.items():
+ description.append_text('Except node %s which is expected to have\n' % str(node_id))
+ description.append_text('\t num tablets: ')
+ expected_tablets.tablets_count.describe_to(description)
+ description.append_text('.')
+ for tablet_id in expected_tablets.tablet_ids:
+ description.append_text('\t tablet with id %s on it' % str(tablet_id))
+
+
+def are_equally_spread_tablets():
+ return TabletsBalanceMatcher().all_balanced()
diff --git a/ydb/tests/library/nemesis/nemesis_core.py b/ydb/tests/library/nemesis/nemesis_core.py
index c992f7f7bf..6886982c23 100644
--- a/ydb/tests/library/nemesis/nemesis_core.py
+++ b/ydb/tests/library/nemesis/nemesis_core.py
@@ -1,80 +1,80 @@
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
import abc
import heapq
import itertools
-import logging
-import random
-import time
+import logging
+import random
+import time
import threading
import six.moves
from six.moves import collections_abc
from ydb.tests.library.nemesis import remote_execution
-
-
-def wrap_in_list(item):
- if isinstance(item, list):
- return item
- else:
- return [item]
-
-
-logger = logging.getLogger(__name__)
-
-
-class RunUnderNemesisContext(object):
- def __init__(self, nemesis_process):
- self.__nemesis = nemesis_process
-
- def __enter__(self):
- if self.__nemesis is not None:
- self.__nemesis.start()
-
- def __exit__(self, type, value, traceback):
- if self.__nemesis is not None:
- self.__nemesis.stop()
+
+
+def wrap_in_list(item):
+ if isinstance(item, list):
+ return item
+ else:
+ return [item]
+
+
+logger = logging.getLogger(__name__)
+
+
+class RunUnderNemesisContext(object):
+ def __init__(self, nemesis_process):
+ self.__nemesis = nemesis_process
+
+ def __enter__(self):
+ if self.__nemesis is not None:
+ self.__nemesis.start()
+
+ def __exit__(self, type, value, traceback):
+ if self.__nemesis is not None:
+ self.__nemesis.stop()
return False
-
+
class NemesisProcess(threading.Thread):
- def __init__(self, nemesis_factory, initial_sleep=10):
- super(NemesisProcess, self).__init__(name='Nemesis')
+ def __init__(self, nemesis_factory, initial_sleep=10):
+ super(NemesisProcess, self).__init__(name='Nemesis')
self.__nemesis_factory = nemesis_factory
self.__private_nemesis_list = wrap_in_list(nemesis_factory)
- self.__pq = PriorityQueue()
+ self.__pq = PriorityQueue()
self.__is_running = threading.Event()
self.__finished_running = threading.Event()
- self.__initial_sleep = initial_sleep
-
- self.daemon = True
+ self.__initial_sleep = initial_sleep
+
+ self.daemon = True
self.__logger = logger.getChild(self.__class__.__name__)
-
- @property
+
+ @property
def __nemesis_list(self):
if self.__private_nemesis_list is None:
self.__private_nemesis_list = wrap_in_list(self.__nemesis_factory())
return self.__private_nemesis_list
-
- def stop(self):
- if self.__is_running.is_set():
- self.__is_running.clear()
- else:
- return
-
+
+ def stop(self):
+ if self.__is_running.is_set():
+ self.__is_running.clear()
+ else:
+ return
+
start_time = time.time()
- self.__logger.info("Stopping Nemesis")
-
- # wait for Nemesis to stop
- finish_time = time.time() + 480
- while not self.__finished_running.is_set() and time.time() < finish_time:
- time.sleep(1)
+ self.__logger.info("Stopping Nemesis")
+
+ # wait for Nemesis to stop
+ finish_time = time.time() + 480
+ while not self.__finished_running.is_set() and time.time() < finish_time:
+ time.sleep(1)
finish_time = time.time()
self.__logger.info(
"Stopped Nemesis successfully in {} seconds".format(
int(finish_time - start_time)
)
)
-
- def run(self):
+
+ def run(self):
try:
self.__run()
except Exception:
@@ -85,13 +85,13 @@ class NemesisProcess(threading.Thread):
def __run(self):
random.seed()
- self.__pq.clear()
- self.__is_running.set()
- self.__finished_running.clear()
-
- time.sleep(self.__initial_sleep)
+ self.__pq.clear()
+ self.__is_running.set()
+ self.__finished_running.clear()
+
+ time.sleep(self.__initial_sleep)
self.__init_pq()
-
+
while self.__is_running.is_set() and self.__pq:
while self.__is_running.is_set() and time.time() < self.__pq.peek_priority():
time.sleep(1)
@@ -101,7 +101,7 @@ class NemesisProcess(threading.Thread):
nemesis = self.__pq.pop()
try:
nemesis.inject_fault()
- except Exception:
+ except Exception:
self.__logger.exception(
'Inject fault for nemesis = {nemesis} failed.'.format(
nemesis=nemesis,
@@ -112,87 +112,87 @@ class NemesisProcess(threading.Thread):
self.__pq.add_task(task=nemesis, priority=priority)
def __init_pq(self):
- self.__logger.info("NemesisProcess started")
+ self.__logger.info("NemesisProcess started")
self.__logger.info("self.nemesis_list = " + str(self.__nemesis_list))
-
- # noinspection PyTypeChecker
+
+ # noinspection PyTypeChecker
for nemesis in self.__nemesis_list:
- prepared = False
- while not prepared:
- try:
- self.__logger.info("Preparing nemesis = " + str(nemesis))
- nemesis.prepare_state()
- prepared = True
- self.__logger.info("Preparation succeeded nemesis = " + str(nemesis))
- except Exception:
- self.__logger.exception("Preparation failed for nemesis = " + str(nemesis))
- time.sleep(1)
-
- # noinspection PyTypeChecker
+ prepared = False
+ while not prepared:
+ try:
+ self.__logger.info("Preparing nemesis = " + str(nemesis))
+ nemesis.prepare_state()
+ prepared = True
+ self.__logger.info("Preparation succeeded nemesis = " + str(nemesis))
+ except Exception:
+ self.__logger.exception("Preparation failed for nemesis = " + str(nemesis))
+ time.sleep(1)
+
+ # noinspection PyTypeChecker
for nemesis in self.__nemesis_list:
- priority = time.time() + nemesis.next_schedule()
- self.__pq.add_task(nemesis, priority=priority)
-
+ priority = time.time() + nemesis.next_schedule()
+ self.__pq.add_task(nemesis, priority=priority)
+
self.__logger.debug("Initial PriorityQueue = " + str(self.__pq))
-
+
def __stop_nemesis(self):
- # Stopping Nemesis
- self.__logger.info("Stopping Nemesis in run()")
- # noinspection PyTypeChecker
+ # Stopping Nemesis
+ self.__logger.info("Stopping Nemesis in run()")
+ # noinspection PyTypeChecker
for nemesis in self.__nemesis_list:
- self.__logger.info("Extracting fault for Nemesis = " + str(nemesis))
+ self.__logger.info("Extracting fault for Nemesis = " + str(nemesis))
try:
nemesis.extract_fault()
except Exception:
logger.exception('Nemesis = {nemesis} extract_fault() failed with exception = '.format(nemesis=nemesis))
-
- self.__finished_running.set()
- self.__logger.info("Stopped Nemesis successfully in run()")
-
-
-class Nemesis(object):
+
+ self.__finished_running.set()
+ self.__logger.info("Stopped Nemesis successfully in run()")
+
+
+class Nemesis(object):
__metaclass__ = abc.ABCMeta
-
- def __init__(self, schedule):
- self.__schedule = Schedule.from_tuple_or_int(schedule)
+
+ def __init__(self, schedule):
+ self.__schedule = Schedule.from_tuple_or_int(schedule)
self.__logger = logging.getLogger(self.__class__.__name__)
-
- def next_schedule(self):
+
+ def next_schedule(self):
"""
Return amount of second from current time this Nemesis should be called with `inject_fault()`
:return: amount of seconds to schedule this Nemesis next time
"""
- return next(self.__schedule)
-
+ return next(self.__schedule)
+
@abc.abstractmethod
- def prepare_state(self):
+ def prepare_state(self):
"""
Prepare state of your Nemesis. Called only once on start.
:return: not specified
"""
- pass
-
+ pass
+
@abc.abstractmethod
- def inject_fault(self):
+ def inject_fault(self):
"""
Inject some fault into running cluster.
:return: not specified
"""
- pass
-
+ pass
+
@abc.abstractmethod
- def extract_fault(self):
+ def extract_fault(self):
"""
Cancel all injected fault if this is possible. Some faults can't be canceled (e.g. node formatting).
Usually called at the end of the Nemesis run.
:return: not specified
"""
- pass
-
+ pass
+
@property
def logger(self):
"""
@@ -205,179 +205,179 @@ class Nemesis(object):
def __str__(self):
return self.__repr__()
- def __repr__(self):
+ def __repr__(self):
return '{ntype}'.format(ntype=type(self))
-
-
-class AbstractNemesisNodeTerrorist(Nemesis):
+
+
+class AbstractNemesisNodeTerrorist(Nemesis):
__metaclass__ = abc.ABCMeta
-
+
def __init__(self, node_list, act_interval, ssh_user=None, remote_sudo_user=None, timeout=60):
- super(AbstractNemesisNodeTerrorist, self).__init__(act_interval)
+ super(AbstractNemesisNodeTerrorist, self).__init__(act_interval)
self._remote_exec_method = remote_execution.execute_command
- self.__nodes = node_list
- self.__ssh_user = ssh_user
- self.__sudo_user = remote_sudo_user
-
+ self.__nodes = node_list
+ self.__ssh_user = ssh_user
+ self.__sudo_user = remote_sudo_user
+
self.__timeout = timeout
-
- def _get_victim_node(self):
- return random.choice(self.__nodes)
-
+
+ def _get_victim_node(self):
+ return random.choice(self.__nodes)
+
@property
def timeout(self):
return self.__timeout
@abc.abstractmethod
- def _commands_on_node(self, node=None):
- pass
-
- def _pre_inject_fault(self, node, commands):
- pass
-
- def _post_inject_fault(self, node, commands):
- pass
-
- def prepare_state(self):
- pass
-
- def extract_fault(self):
- pass
-
- def inject_fault(self):
- success = True
- node = random.choice(self.__nodes)
- node_commands = wrap_in_list(self._commands_on_node(node))
-
- self._pre_inject_fault(node, node_commands)
-
- for command in node_commands:
+ def _commands_on_node(self, node=None):
+ pass
+
+ def _pre_inject_fault(self, node, commands):
+ pass
+
+ def _post_inject_fault(self, node, commands):
+ pass
+
+ def prepare_state(self):
+ pass
+
+ def extract_fault(self):
+ pass
+
+ def inject_fault(self):
+ success = True
+ node = random.choice(self.__nodes)
+ node_commands = wrap_in_list(self._commands_on_node(node))
+
+ self._pre_inject_fault(node, node_commands)
+
+ for command in node_commands:
cmd_result = self._remote_exec_method(self._full_command(node, command))
- success = cmd_result == 0 and success
-
- self._post_inject_fault(node, node_commands)
-
- return success
-
- def _full_command(self, node, node_command):
- if self.__ssh_user is not None:
- address = "%s@%s" % (self.__ssh_user, node)
- else:
- address = node
- if self.__sudo_user is not None:
- if self.__sudo_user == 'root':
- return ["ssh", address, "sudo"] + list(node_command)
- else:
- return [
- "ssh", address,
- "sudo", "-u", self.__sudo_user
- ] + list(node_command)
- else:
- return ["ssh", address] + list(node_command)
-
-
+ success = cmd_result == 0 and success
+
+ self._post_inject_fault(node, node_commands)
+
+ return success
+
+ def _full_command(self, node, node_command):
+ if self.__ssh_user is not None:
+ address = "%s@%s" % (self.__ssh_user, node)
+ else:
+ address = node
+ if self.__sudo_user is not None:
+ if self.__sudo_user == 'root':
+ return ["ssh", address, "sudo"] + list(node_command)
+ else:
+ return [
+ "ssh", address,
+ "sudo", "-u", self.__sudo_user
+ ] + list(node_command)
+ else:
+ return ["ssh", address] + list(node_command)
+
+
class Schedule(collections_abc.Iterator):
- def __init__(self, schedule_iterator):
- self.__iterator = schedule_iterator
-
- def __iter__(self):
- return self
-
+ def __init__(self, schedule_iterator):
+ self.__iterator = schedule_iterator
+
+ def __iter__(self):
+ return self
+
def __next__(self):
return next(self.__iterator)
- def next(self):
- return next(self.__iterator)
-
- @staticmethod
- def combine(list_of_iterators):
- return Schedule(itertools.chain.from_iterable(list_of_iterators))
-
- @staticmethod
- def cycle(factory_of_list_of_iterators):
- def iterable():
- while True:
- list_of_iterators = factory_of_list_of_iterators()
- for item in list_of_iterators:
- yield item
-
- return Schedule(itertools.chain.from_iterable(iterable()))
-
- @staticmethod
- def from_tuple_or_int(value):
- if isinstance(value, Schedule):
- return value
- elif isinstance(value, int):
- return Schedule(itertools.repeat(value))
- else:
- a, b = value
+ def next(self):
+ return next(self.__iterator)
+
+ @staticmethod
+ def combine(list_of_iterators):
+ return Schedule(itertools.chain.from_iterable(list_of_iterators))
+
+ @staticmethod
+ def cycle(factory_of_list_of_iterators):
+ def iterable():
+ while True:
+ list_of_iterators = factory_of_list_of_iterators()
+ for item in list_of_iterators:
+ yield item
+
+ return Schedule(itertools.chain.from_iterable(iterable()))
+
+ @staticmethod
+ def from_tuple_or_int(value):
+ if isinstance(value, Schedule):
+ return value
+ elif isinstance(value, int):
+ return Schedule(itertools.repeat(value))
+ else:
+ a, b = value
return Schedule(six.moves.map(lambda x: random.randint(a, b), itertools.count()))
-
-
+
+
class RunOnceSchedule(Schedule):
def __init__(self, time_at):
super(RunOnceSchedule, self).__init__(iter([time_at, 2**50]))
-class PriorityQueue(object):
- def __init__(self):
- super(PriorityQueue, self).__init__()
- self.__heap = []
- self.__count = itertools.count()
-
- def add_task(self, task, priority):
- entry = priority, next(self.__count), task
- heapq.heappush(self.__heap, entry)
-
- def pop(self):
- _, _, task = heapq.heappop(self.__heap)
- return task
-
- def peek(self):
- _, _, task = self.__heap[0]
- return task
-
- def peek_priority(self):
- priority, _, _ = self.__heap[0]
- return priority
-
- def clear(self):
- self.__heap = []
-
- def __len__(self):
- return len(self.__heap)
-
- def __str__(self):
+class PriorityQueue(object):
+ def __init__(self):
+ super(PriorityQueue, self).__init__()
+ self.__heap = []
+ self.__count = itertools.count()
+
+ def add_task(self, task, priority):
+ entry = priority, next(self.__count), task
+ heapq.heappush(self.__heap, entry)
+
+ def pop(self):
+ _, _, task = heapq.heappop(self.__heap)
+ return task
+
+ def peek(self):
+ _, _, task = self.__heap[0]
+ return task
+
+ def peek_priority(self):
+ priority, _, _ = self.__heap[0]
+ return priority
+
+ def clear(self):
+ self.__heap = []
+
+ def __len__(self):
+ return len(self.__heap)
+
+ def __str__(self):
# return str(self.__heap)
return '\n'.join(map(str, sorted([(entry[0], entry[1]) for entry in self.__heap])))
-
-
-class FakeNemesis(Nemesis):
+
+
+class FakeNemesis(Nemesis):
"""
>>> n = FakeNemesis(None)
>>> print(n.logger.name)
FakeNemesis
"""
-
+
def __init__(self, out_queue, schedule=1):
super(FakeNemesis, self).__init__(schedule=schedule)
self.__out_queue = out_queue
-
- def inject_fault(self):
+
+ def inject_fault(self):
self.__out_queue.put('inject_fault')
-
- def prepare_state(self):
+
+ def prepare_state(self):
self.__out_queue.put('prepare_state')
-
- def extract_fault(self):
+
+ def extract_fault(self):
self.__out_queue.put('extract_fault')
-
- def next_schedule(self):
+
+ def next_schedule(self):
n = super(FakeNemesis, self).next_schedule()
self.__out_queue.put('next_schedule = ' + str(n))
return n
-
-
+
+
class ExceptionThrowingNemesis(Nemesis):
def __init__(self, out_queue, schedule=1):
@@ -400,54 +400,54 @@ class ExceptionThrowingNemesis(Nemesis):
return n
-def test_priority_queue():
- """
- >>> pq = PriorityQueue()
+def test_priority_queue():
+ """
+ >>> pq = PriorityQueue()
>>> pq.add_task('last', time.time() + 10)
>>> pq.add_task('first', time.time())
>>> pq.add_task('task 4', time.time() + 4)
>>> pq.add_task('task 3', time.time() + 3)
>>> pq.add_task('task 2', time.time() + 2)
- >>> pq.pop()
+ >>> pq.pop()
'first'
- >>> pq.peek()
+ >>> pq.peek()
'task 2'
- >>> len(pq)
- 4
- >>> pq.pop()
+ >>> len(pq)
+ 4
+ >>> pq.pop()
'task 2'
- >>> len(pq)
- 3
- >>> bool(pq)
- True
+ >>> len(pq)
+ 3
+ >>> bool(pq)
+ True
>>> pq.add_task('first', time.time())
- >>> pq.pop()
+ >>> pq.pop()
'first'
- >>> pq.pop()
+ >>> pq.pop()
'task 3'
- >>> pq.pop()
+ >>> pq.pop()
'task 4'
>>> pq.pop()
'last'
- >>> bool(pq)
- False
- """
-
-
-def test_nemesis_process():
- """
- >>> from multiprocessing import Queue
+ >>> bool(pq)
+ False
+ """
+
+
+def test_nemesis_process():
+ """
+ >>> from multiprocessing import Queue
>>> timeout = 10
- >>> out_queue = Queue()
+ >>> out_queue = Queue()
>>> np = NemesisProcess(FakeNemesis(out_queue, schedule=1), initial_sleep=0)
- >>> np.start()
- >>> time.sleep(1)
+ >>> np.start()
+ >>> time.sleep(1)
>>> out_queue.get(timeout=timeout)
- 'prepare_state'
+ 'prepare_state'
>>> out_queue.get(timeout=timeout)
'next_schedule = 1'
>>> out_queue.get(timeout=timeout)
- 'inject_fault'
+ 'inject_fault'
>>> out_queue.get(timeout=timeout)
'next_schedule = 1'
>>> out_queue.get(timeout=timeout)
@@ -457,25 +457,25 @@ def test_nemesis_process():
>>> np.stop()
>>> out_queue.get(timeout=timeout)
- 'extract_fault'
+ 'extract_fault'
>>> np = NemesisProcess(ExceptionThrowingNemesis(out_queue, schedule=1), initial_sleep=0)
>>> np.start()
>>> time.sleep(3)
>>> np.stop()
- """
-
-
-def test_schedule():
- """
- >>> random.seed(123)
- >>> s = Schedule.from_tuple_or_int((10, 20))
- >>> [next(s) for _ in range(10)]
+ """
+
+
+def test_schedule():
+ """
+ >>> random.seed(123)
+ >>> s = Schedule.from_tuple_or_int((10, 20))
+ >>> [next(s) for _ in range(10)]
[10, 14, 11, 16, 14, 11, 10, 16, 18, 18]
- >>> s = Schedule.from_tuple_or_int(15)
- >>> [next(s) for _ in range(10)]
- [15, 15, 15, 15, 15, 15, 15, 15, 15, 15]
- >>> q = Schedule.combine(iter([[1, 2], Schedule.from_tuple_or_int(5)]))
- >>> [next(q) for _ in range(10)]
- [1, 2, 5, 5, 5, 5, 5, 5, 5, 5]
- """
+ >>> s = Schedule.from_tuple_or_int(15)
+ >>> [next(s) for _ in range(10)]
+ [15, 15, 15, 15, 15, 15, 15, 15, 15, 15]
+ >>> q = Schedule.combine(iter([[1, 2], Schedule.from_tuple_or_int(5)]))
+ >>> [next(q) for _ in range(10)]
+ [1, 2, 5, 5, 5, 5, 5, 5, 5, 5]
+ """
diff --git a/ydb/tests/library/nemesis/nemesis_process_killers.py b/ydb/tests/library/nemesis/nemesis_process_killers.py
index df8e88be1b..2ec73a7b35 100644
--- a/ydb/tests/library/nemesis/nemesis_process_killers.py
+++ b/ydb/tests/library/nemesis/nemesis_process_killers.py
@@ -1,69 +1,69 @@
# -*- coding: utf-8 -*-
-import logging
+import logging
import random
from ydb.tests.library.nemesis.nemesis_core import AbstractNemesisNodeTerrorist
from ydb.tests.library.nemesis.remote_execution import execute_command_with_output
-
+
logger = logging.getLogger(__name__)
-
-DEFAULT_KILL_ALL_CHANCE = 0.3
-
-
-def kill_cmd(process, signal):
- return (
- "killall",
- "-s",
- str(signal),
- str(process)
- )
-
-
-def kill_pid_cmd(pid, signal):
- return (
- 'kill',
- '-s',
- str(signal),
- str(pid)
- )
-
-
-class NemesisProcessKiller(AbstractNemesisNodeTerrorist):
- def __init__(
- self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None,
- kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, signal='SIGKILL', processes_filters=None,
- timeout=3.0
- ):
- super(NemesisProcessKiller, self).__init__(cluster, act_interval, timeout=timeout,
- ssh_user=ssh_user, remote_sudo_user=remote_sudo_user)
- self.__victims_procs = processes_affected
+
+DEFAULT_KILL_ALL_CHANCE = 0.3
+
+
+def kill_cmd(process, signal):
+ return (
+ "killall",
+ "-s",
+ str(signal),
+ str(process)
+ )
+
+
+def kill_pid_cmd(pid, signal):
+ return (
+ 'kill',
+ '-s',
+ str(signal),
+ str(pid)
+ )
+
+
+class NemesisProcessKiller(AbstractNemesisNodeTerrorist):
+ def __init__(
+ self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None,
+ kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, signal='SIGKILL', processes_filters=None,
+ timeout=3.0
+ ):
+ super(NemesisProcessKiller, self).__init__(cluster, act_interval, timeout=timeout,
+ ssh_user=ssh_user, remote_sudo_user=remote_sudo_user)
+ self.__victims_procs = processes_affected
self.__kill_all_chance = kill_everything_chance
- self._signal = signal
-
- self._command_for_victim = kill_cmd if processes_filters is None else kill_pid_cmd
- self.__process_filters = processes_filters
-
- def _commands_on_node(self, node=None):
- ret = []
- victim_candidates = self._get_victim(node)
-
- if random.random() < self.__kill_all_chance or not victim_candidates:
- real_victims = victim_candidates
- else:
- real_victims = [random.choice(victim_candidates), ]
-
- for prc in real_victims:
- ret.append(
- self._command_for_victim(prc, self._signal)
- )
- return ret
-
- def _get_victim(self, node=None):
- if self.__process_filters is None:
- return self.__victims_procs
- else:
- return self.__get_victims_by_filter(node)
-
- def __get_victims_by_filter(self, node):
+ self._signal = signal
+
+ self._command_for_victim = kill_cmd if processes_filters is None else kill_pid_cmd
+ self.__process_filters = processes_filters
+
+ def _commands_on_node(self, node=None):
+ ret = []
+ victim_candidates = self._get_victim(node)
+
+ if random.random() < self.__kill_all_chance or not victim_candidates:
+ real_victims = victim_candidates
+ else:
+ real_victims = [random.choice(victim_candidates), ]
+
+ for prc in real_victims:
+ ret.append(
+ self._command_for_victim(prc, self._signal)
+ )
+ return ret
+
+ def _get_victim(self, node=None):
+ if self.__process_filters is None:
+ return self.__victims_procs
+ else:
+ return self.__get_victims_by_filter(node)
+
+ def __get_victims_by_filter(self, node):
ps_command = ['ps', 'ax']
result, output = execute_command_with_output(self._full_command(node, ps_command), timeout=self.timeout)
if result:
@@ -74,80 +74,80 @@ class NemesisProcessKiller(AbstractNemesisNodeTerrorist):
lambda x: x.strip().split(' ')[0],
self.__filter_output(output)
)
-
- def __filter_output(self, output):
- lines_match = {}
- for i in range(len(output)):
- lines_match[i] = True
-
- for filt in self.__process_filters:
- for line_number, still_matching in lines_match.items():
- if still_matching:
- if filt not in output[line_number]:
- lines_match[line_number] = False
-
- return [output[i] for i in lines_match if lines_match[i]]
-
-
-class NemesisProcessSuspender(NemesisProcessKiller):
- def __init__(
- self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None,
- kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, processes_filters=None
- ):
-
+
+ def __filter_output(self, output):
+ lines_match = {}
+ for i in range(len(output)):
+ lines_match[i] = True
+
+ for filt in self.__process_filters:
+ for line_number, still_matching in lines_match.items():
+ if still_matching:
+ if filt not in output[line_number]:
+ lines_match[line_number] = False
+
+ return [output[i] for i in lines_match if lines_match[i]]
+
+
+class NemesisProcessSuspender(NemesisProcessKiller):
+ def __init__(
+ self, cluster, processes_affected, act_interval, ssh_user=None, remote_sudo_user=None,
+ kill_everything_chance=DEFAULT_KILL_ALL_CHANCE, processes_filters=None
+ ):
+
super(NemesisProcessSuspender, self).__init__(
- cluster, processes_affected, act_interval, ssh_user, remote_sudo_user,
- kill_everything_chance, signal='SIGSTOP', processes_filters=processes_filters
+ cluster, processes_affected, act_interval, ssh_user, remote_sudo_user,
+ kill_everything_chance, signal='SIGSTOP', processes_filters=processes_filters
)
- self.__to_unsuspend = []
-
- def __wake_all_suspended(self):
- for node, commands in self.__to_unsuspend:
- for command in commands:
+ self.__to_unsuspend = []
+
+ def __wake_all_suspended(self):
+ for node, commands in self.__to_unsuspend:
+ for command in commands:
self._remote_exec_method(self._full_command(node, command), timeout=self.timeout)
- self.__to_unsuspend = []
-
- def _pre_inject_fault(self, node, commands):
- self.__wake_all_suspended()
- wake_commands = [self._command_for_victim(i[-1], signal="SIGCONT") for i in commands]
- self.__to_unsuspend.append((node, wake_commands))
-
- def extract_fault(self):
- self.__wake_all_suspended()
-
-
-def fake_execute_command(command):
- print("{}".format(command))
-
-
+ self.__to_unsuspend = []
+
+ def _pre_inject_fault(self, node, commands):
+ self.__wake_all_suspended()
+ wake_commands = [self._command_for_victim(i[-1], signal="SIGCONT") for i in commands]
+ self.__to_unsuspend.append((node, wake_commands))
+
+ def extract_fault(self):
+ self.__wake_all_suspended()
+
+
+def fake_execute_command(command):
+ print("{}".format(command))
+
+
class TestNemesisProcessSuspender(NemesisProcessSuspender):
- _remote_exec_method = fake_execute_command
+ _remote_exec_method = fake_execute_command
"""
>>> random.seed(123)
>>> n = TestNemesisProcessSuspender(range(8), range(2), 1)
>>> n.prepare_state()
>>> n.inject_fault()
- ['ssh', 0, 'killall', '-s', 'SIGSTOP', '0']
- ['ssh', 0, 'killall', '-s', 'SIGSTOP', '1']
+ ['ssh', 0, 'killall', '-s', 'SIGSTOP', '0']
+ ['ssh', 0, 'killall', '-s', 'SIGSTOP', '1']
>>> n.inject_fault()
- ['ssh', 0, 'killall', '-s', 'SIGCONT', '0']
- ['ssh', 0, 'killall', '-s', 'SIGCONT', '1']
- ['ssh', 3, 'killall', '-s', 'SIGSTOP', '0']
- ['ssh', 3, 'killall', '-s', 'SIGSTOP', '1']
+ ['ssh', 0, 'killall', '-s', 'SIGCONT', '0']
+ ['ssh', 0, 'killall', '-s', 'SIGCONT', '1']
+ ['ssh', 3, 'killall', '-s', 'SIGSTOP', '0']
+ ['ssh', 3, 'killall', '-s', 'SIGSTOP', '1']
>>> n.inject_fault()
- ['ssh', 3, 'killall', '-s', 'SIGCONT', '0']
- ['ssh', 3, 'killall', '-s', 'SIGCONT', '1']
- ['ssh', 7, 'killall', '-s', 'SIGSTOP', '0']
- ['ssh', 7, 'killall', '-s', 'SIGSTOP', '1']
+ ['ssh', 3, 'killall', '-s', 'SIGCONT', '0']
+ ['ssh', 3, 'killall', '-s', 'SIGCONT', '1']
+ ['ssh', 7, 'killall', '-s', 'SIGSTOP', '0']
+ ['ssh', 7, 'killall', '-s', 'SIGSTOP', '1']
>>> n.inject_fault()
- ['ssh', 7, 'killall', '-s', 'SIGCONT', '0']
- ['ssh', 7, 'killall', '-s', 'SIGCONT', '1']
- ['ssh', 4, 'killall', '-s', 'SIGSTOP', '1']
+ ['ssh', 7, 'killall', '-s', 'SIGCONT', '0']
+ ['ssh', 7, 'killall', '-s', 'SIGCONT', '1']
+ ['ssh', 4, 'killall', '-s', 'SIGSTOP', '1']
>>> n.inject_fault()
- ['ssh', 4, 'killall', '-s', 'SIGCONT', '1']
- ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0']
+ ['ssh', 4, 'killall', '-s', 'SIGCONT', '1']
+ ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0']
>>> n.inject_fault()
- ['ssh', 1, 'killall', '-s', 'SIGCONT', '0']
- ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0']
- ['ssh', 1, 'killall', '-s', 'SIGSTOP', '1']
+ ['ssh', 1, 'killall', '-s', 'SIGCONT', '0']
+ ['ssh', 1, 'killall', '-s', 'SIGSTOP', '0']
+ ['ssh', 1, 'killall', '-s', 'SIGSTOP', '1']
"""
diff --git a/ydb/tests/library/nemesis/nemesis_time_terrorist.py b/ydb/tests/library/nemesis/nemesis_time_terrorist.py
index fb32d01c5f..564b721bfb 100644
--- a/ydb/tests/library/nemesis/nemesis_time_terrorist.py
+++ b/ydb/tests/library/nemesis/nemesis_time_terrorist.py
@@ -1,46 +1,46 @@
-# -*- coding: utf-8 -*-
-from datetime import datetime, timedelta
-import logging
-from random import randint
-
+# -*- coding: utf-8 -*-
+from datetime import datetime, timedelta
+import logging
+from random import randint
+
from ydb.tests.library.nemesis.nemesis_core import AbstractNemesisNodeTerrorist
from ydb.tests.library.nemesis.remote_execution import execute_command_with_output
-
-logger = logging.getLogger(__name__)
-
-
-class NemesisTimeChanger(AbstractNemesisNodeTerrorist):
- def __init__(
- self, cluster, act_interval, ssh_user=None, remote_sudo_user=None,
- time_change_range=(-60, 60), timeout=3.0
- ):
- super(NemesisTimeChanger, self).__init__(
- cluster, act_interval, ssh_user=ssh_user, remote_sudo_user=remote_sudo_user, timeout=timeout
- )
- self.__change_range = time_change_range
- self.__get_time_cmd = ('date', '+%s')
-
- @staticmethod
- def set_time_cmd(time_to_set):
- time_str = '{:02d}:{:02d}:{:02d}'.format(time_to_set.hour, time_to_set.minute, time_to_set.second)
- return (
- 'date', '+%T', '-s', time_str
- )
-
- def _commands_on_node(self, node=None):
- current_time = self.__get_curr_time(node)
- if current_time is None:
- return []
- else:
- delta = timedelta(seconds=randint(*self.__change_range))
- return self.set_time_cmd(current_time + delta)
-
- def __get_curr_time(self, node):
- result, output = execute_command_with_output(
+
+logger = logging.getLogger(__name__)
+
+
+class NemesisTimeChanger(AbstractNemesisNodeTerrorist):
+ def __init__(
+ self, cluster, act_interval, ssh_user=None, remote_sudo_user=None,
+ time_change_range=(-60, 60), timeout=3.0
+ ):
+ super(NemesisTimeChanger, self).__init__(
+ cluster, act_interval, ssh_user=ssh_user, remote_sudo_user=remote_sudo_user, timeout=timeout
+ )
+ self.__change_range = time_change_range
+ self.__get_time_cmd = ('date', '+%s')
+
+ @staticmethod
+ def set_time_cmd(time_to_set):
+ time_str = '{:02d}:{:02d}:{:02d}'.format(time_to_set.hour, time_to_set.minute, time_to_set.second)
+ return (
+ 'date', '+%T', '-s', time_str
+ )
+
+ def _commands_on_node(self, node=None):
+ current_time = self.__get_curr_time(node)
+ if current_time is None:
+ return []
+ else:
+ delta = timedelta(seconds=randint(*self.__change_range))
+ return self.set_time_cmd(current_time + delta)
+
+ def __get_curr_time(self, node):
+ result, output = execute_command_with_output(
self._full_command(node, self.__get_time_cmd), timeout=self.timeout
- )
- if result:
- logger.error("Failed to fetch victim process on node {node}. Will not act this time".format(node=node))
- return None
- else:
- return datetime.fromtimestamp(int(output[0].strip()))
+ )
+ if result:
+ logger.error("Failed to fetch victim process on node {node}. Will not act this time".format(node=node))
+ return None
+ else:
+ return datetime.fromtimestamp(int(output[0].strip()))
diff --git a/ydb/tests/library/nemesis/remote_execution.py b/ydb/tests/library/nemesis/remote_execution.py
index 15e6827424..b7b5be9c68 100644
--- a/ydb/tests/library/nemesis/remote_execution.py
+++ b/ydb/tests/library/nemesis/remote_execution.py
@@ -9,19 +9,19 @@ from concurrent import futures
logger = logging.getLogger(__name__)
-def execute_command(command, timeout=60):
+def execute_command(command, timeout=60):
logger.info("Running: {}".format(command))
process = subprocess.Popen(command)
- wait_timeout(process, timeout)
- return process.returncode
-
-
+ wait_timeout(process, timeout)
+ return process.returncode
+
+
def execute_command_with_output(command, timeout=60):
logger.info("Running command = {}".format(command))
list_of_lines = []
with tempfile.TemporaryFile() as f_out, tempfile.TemporaryFile() as f_err:
process = subprocess.Popen(command, stdout=f_out, stderr=f_err)
- wait_timeout(process, timeout)
+ wait_timeout(process, timeout)
process_return_code = process.returncode
f_err.flush()
f_err.seek(0)
@@ -30,22 +30,22 @@ def execute_command_with_output(command, timeout=60):
f_out.flush()
f_out.seek(0)
list_of_lines = list(f_out.readlines())
-
- logger.info("Finished execution command = {}".format(command))
+
+ logger.info("Finished execution command = {}".format(command))
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Command output = \n{}'.format('\n'.join(list_of_lines)))
logger.debug('Stderr = \n{}'.format('\n'.join(std_err_lines)))
return process_return_code, list_of_lines
-
-def wait_timeout(process, timeout):
- start_time = time.time()
- while time.time() < start_time + timeout:
- process.poll()
- if process.returncode is not None:
- return
- time.sleep(0.5)
- process.kill()
+
+def wait_timeout(process, timeout):
+ start_time = time.time()
+ while time.time() < start_time + timeout:
+ process.poll()
+ if process.returncode is not None:
+ return
+ time.sleep(0.5)
+ process.kill()
process.wait()
diff --git a/ydb/tests/library/nemesis/safety_warden.py b/ydb/tests/library/nemesis/safety_warden.py
index 51822ed0a1..e9d43a0c38 100644
--- a/ydb/tests/library/nemesis/safety_warden.py
+++ b/ydb/tests/library/nemesis/safety_warden.py
@@ -1,48 +1,48 @@
-# -*- coding: utf-8 -*-
-
+# -*- coding: utf-8 -*-
+
import functools
import itertools
-import logging
-from abc import ABCMeta, abstractmethod
+import logging
+from abc import ABCMeta, abstractmethod
from ydb.tests.library.nemesis.remote_execution import execute_command_with_output_on_hosts
-
-
-logger = logging.getLogger()
-
-
-class SafetyWarden(object):
- __metaclass__ = ABCMeta
-
+
+
+logger = logging.getLogger()
+
+
+class SafetyWarden(object):
+ __metaclass__ = ABCMeta
+
def __init__(self, name):
super(SafetyWarden, self).__init__()
self.__name = name
- @abstractmethod
- def list_of_safety_violations(self):
- pass
-
+ @abstractmethod
+ def list_of_safety_violations(self):
+ pass
+
@property
- def name(self):
+ def name(self):
return self.__name
-
+
def __str__(self):
return self.name
-
-class AggregateSafetyWarden(SafetyWarden):
- def __init__(self, list_of_safety_wardens):
+
+class AggregateSafetyWarden(SafetyWarden):
+ def __init__(self, list_of_safety_wardens):
super(AggregateSafetyWarden, self).__init__('AggregateSafetyWarden')
self.__list_of_safety_wardens = list(list_of_safety_wardens)
-
- def list_of_safety_violations(self):
- all_safety_violations = []
+
+ def list_of_safety_violations(self):
+ all_safety_violations = []
for warden in self.__list_of_safety_wardens:
- all_safety_violations.extend(
- warden.list_of_safety_violations()
- )
- return all_safety_violations
-
-
+ all_safety_violations.extend(
+ warden.list_of_safety_violations()
+ )
+ return all_safety_violations
+
+
def split_in_chunks(list_of_lines, chunk_size):
"""
>>> a = list(map(str, range(6)))
@@ -132,7 +132,7 @@ class GrepLogFileForMarkers(AbstractRemoteCommandExecutionSafetyWarden):
+ [
log_file_name,
]
- )
+ )
remote_command.extend(
[