diff options
| author | komels <[email protected]> | 2022-04-15 16:53:39 +0300 |
|---|---|---|
| committer | komels <[email protected]> | 2022-04-15 16:53:39 +0300 |
| commit | 703a2fb6e100d202d1c7fcd052d73bd5affef408 (patch) | |
| tree | 22b7320c06bb04d86dbf7b9af9ae44281331cd15 /kikimr/persqueue/sdk/deprecated/cpp | |
| parent | 3375bbfda1e2afb03aa2072bf5f2f2c3a26026e8 (diff) | |
Move 'kikimr/yndx'-depending tests out of ydb/core
ref:0a380e13308d579e0545a76924330d1ca5129c43
Diffstat (limited to 'kikimr/persqueue/sdk/deprecated/cpp')
89 files changed, 0 insertions, 14423 deletions
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt b/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt deleted file mode 100644 index 44cb2496eda..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ - -# This file was gererated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(deprecated-cpp-v2) -target_link_libraries(deprecated-cpp-v2 PUBLIC - contrib-libs-cxxsupp - yutil - contrib-libs-grpc - contrib-libs-protobuf - cpp-client-iam - cpp-client-ydb_persqueue - api-grpc-yndx - api-protos-yndx - yndx-persqueue-read_batch_converter - library-cpp-blockcodecs - cpp-containers-disjoint_interval_tree - cpp-containers-intrusive_rb_tree - cpp-grpc-common - cpp-http-simple - library-cpp-json - library-cpp-logger - cpp-streams-lzop - cpp-streams-zstd - cpp-string_utils-quote - cpp-threading-future - cpp-tvmauth-client - library-persqueue-topic_parser_public - cpp-client-resources -) -target_sources(deprecated-cpp-v2 PRIVATE - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp -) diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h deleted file mode 100644 index 623a293dc3c..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once -#include "fake_actor.h" - -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/actorid.h> -#include <library/cpp/actors/core/actorsystem.h> -#include <library/cpp/actors/core/events.h> - -namespace NPersQueue { - -struct IActorInterface { - virtual ~IActorInterface() = default; - - // Sender of all messages from PQLib objects. - virtual NActors::TActorId GetActorID() const noexcept = 0; -}; - -class TActorHolder : public IActorInterface -{ -protected: - TActorHolder(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID) - : ActorSystem(actorSystem) - , ParentActorID(parentActorID) - { - ActorID = ActorSystem->Register(new TFakeActor()); - } - - ~TActorHolder() { - ActorSystem->Send(new NActors::IEventHandle(ActorID, ActorID, new NActors::TEvents::TEvPoisonPill())); - } - - template <class TResponseEvent> - void Subscribe(ui64 requestId, NThreading::TFuture<typename TResponseEvent::TResponseType>&& future) { - auto handler = [ - requestId = requestId, - actorSystem = ActorSystem, - actorID = ActorID, - parentActorID = ParentActorID - ](const NThreading::TFuture<typename TResponseEvent::TResponseType>& future) { - actorSystem->Send(new NActors::IEventHandle( - parentActorID, - actorID, - new TResponseEvent(typename TResponseEvent::TResponseType(future.GetValue()), requestId) - )); - }; - future.Subscribe(handler); - } - -public: - NActors::TActorId GetActorID() const noexcept override { - return ActorID; - } - -protected: - NActors::TActorSystem* ActorSystem; - NActors::TActorId ParentActorID; - NActors::TActorId ActorID; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h deleted file mode 100644 index ce2f7682c1e..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h +++ /dev/null @@ -1,108 +0,0 @@ -#pragma once -#include "actor_interface.h" -#include "responses.h" - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> - -#include <util/datetime/base.h> - -namespace NPersQueue { - -template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId> -class TProducerActorWrapper: public TActorHolder { -public: - using TObjectIsDeadEvent = TPQLibResponseEvent<TError, TObjectIsDeadEventId>; - using TCreateResponseEvent = TPQLibResponseEvent<TProducerCreateResponse, TCreateResponseEventId>; - using TCommitResponseEvent = TPQLibResponseEvent<TProducerCommitResponse, TCommitResponseEventId>; - -public: - TProducerActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IProducer> producer) - : TActorHolder(actorSystem, parentActorID) - , Producer(std::move(producer)) - { - } - - void Start(TInstant deadline, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { - Subscribe<TCreateResponseEvent>(requestId, Producer->Start(deadline)); - Subscribe<TObjectIsDeadEvent>(isDeadRequestId, Producer->IsDead()); - } - - void Start(TDuration timeout, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { - Start(TInstant::Now() + timeout, requestId, isDeadRequestId); - } - - void Write(TProducerSeqNo seqNo, TData data, ui64 requestId = 0) noexcept { - Subscribe<TCommitResponseEvent>(requestId, Producer->Write(seqNo, std::move(data))); - } - - void Write(TData data, ui64 requestId = 0) noexcept { - Subscribe<TCommitResponseEvent>(requestId, Producer->Write(std::move(data))); - } - -private: - THolder<IProducer> Producer; -}; - -template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TGetMessageEventId> -class TConsumerActorWrapper: public TActorHolder { -public: - using TObjectIsDeadEvent = TPQLibResponseEvent<TError, TObjectIsDeadEventId>; - using TCreateResponseEvent = TPQLibResponseEvent<TConsumerCreateResponse, TCreateResponseEventId>; - using TGetMessageEvent = TPQLibResponseEvent<TConsumerMessage, TGetMessageEventId>; - -public: - TConsumerActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IConsumer> consumer) - : TActorHolder(actorSystem, parentActorID) - , Consumer(std::move(consumer)) - { - } - - void Start(TInstant deadline, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { - Subscribe<TCreateResponseEvent>(requestId, Consumer->Start(deadline)); - Subscribe<TObjectIsDeadEvent>(isDeadRequestId, Consumer->IsDead()); - } - - void Start(TDuration timeout, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { - Start(TInstant::Now() + timeout, requestId, isDeadRequestId); - } - - void GetNextMessage(ui64 requestId = 0) noexcept { - Subscribe<TGetMessageEvent>(requestId, Consumer->GetNextMessage()); - } - - void Commit(const TVector<ui64>& cookies) noexcept { - Consumer->Commit(cookies); // no future in PQLib API - } - - void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { - Consumer->RequestPartitionStatus(topic, partition, generation); // no future in PQLib API - } - -private: - THolder<IConsumer> Consumer; -}; - -template <ui32 TOriginDataEventId> -class TProcessorActorWrapper: public TActorHolder { -public: - using TOriginDataEvent = TPQLibResponseEvent<TOriginData, TOriginDataEventId>; - -public: - TProcessorActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IProcessor> processor) - : TActorHolder(actorSystem, parentActorID) - , Processor(std::move(processor)) - { - } - - void GetNextData(ui64 requestId = 0) noexcept { - Subscribe<TOriginDataEvent>(requestId, Processor->GetNextData()); - } - -private: - THolder<IProcessor> Processor; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp deleted file mode 100644 index 1e08597ec3e..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp +++ /dev/null @@ -1,119 +0,0 @@ -#include "actor_wrappers.h" -#include "responses.h" - -#include <ydb/core/testlib/basics/runtime.h> -#include <ydb/core/testlib/basics/appdata.h> - -#include <library/cpp/actors/core/events.h> -#include <library/cpp/testing/gmock_in_unittest/gmock.h> -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/thread.h> - -using namespace testing; - -namespace NPersQueue { - -namespace TEvPQLibTests { - enum EEv { - EvPQLibObjectIsDead = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), - EvPQLibProducerCreateResponse, - EvPQLibProducerCommitResponse, - EvEnd - }; -} // TEvPQLibTests - -class TMockProducer: public IProducer { -public: - NThreading::TFuture<TProducerCreateResponse> Start(TInstant) noexcept override { - return NThreading::MakeFuture<TProducerCreateResponse>(MockStart()); - } - - NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo, TData data) noexcept override { - return Write(std::move(data)); - } - - NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept override { - return NThreading::MakeFuture<TProducerCommitResponse>(MockWrite(data)); - } - - NThreading::TFuture<TError> IsDead() noexcept override { - return MockIsDead(); - } - - MOCK_METHOD(TProducerCreateResponse, MockStart, (), ()); - MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ()); - MOCK_METHOD(TProducerCommitResponse, MockWrite, (TData), ()); -}; - -Y_UNIT_TEST_SUITE(TProducerActorWrapperTest) { - Y_UNIT_TEST(PassesEvents) { - NActors::TTestActorRuntime runtime; - runtime.Initialize(NKikimr::TAppPrepare().Unwrap()); - auto edgeActorId = runtime.AllocateEdgeActor(); - THolder<TMockProducer> producerHolder = MakeHolder<TMockProducer>(); - TMockProducer& producer = *producerHolder; - - // expectations - TWriteResponse startResponse; - startResponse.MutableInit()->SetMaxSeqNo(42); - EXPECT_CALL(producer, MockStart()) - .WillOnce(Return(TProducerCreateResponse(std::move(startResponse)))); - - NThreading::TPromise<TError> deadPromise = NThreading::NewPromise<TError>(); - EXPECT_CALL(producer, MockIsDead()) - .WillOnce(Return(deadPromise.GetFuture())); - - TWriteResponse writeResponse; - writeResponse.MutableAck()->SetSeqNo(100); - TData data("data"); - EXPECT_CALL(producer, MockWrite(data)) - .WillOnce(Return(TProducerCommitResponse(100, data, std::move(writeResponse)))); - - // wrapper creation - using TProducerType = TProducerActorWrapper< - TEvPQLibTests::EvPQLibObjectIsDead, - TEvPQLibTests::EvPQLibProducerCreateResponse, - TEvPQLibTests::EvPQLibProducerCommitResponse - >; - - THolder<TProducerType> wrapper = MakeHolder<TProducerType>(runtime.GetAnyNodeActorSystem(), edgeActorId, std::move(producerHolder)); - - // checks - auto actorId = wrapper->GetActorID(); - UNIT_ASSERT(actorId); - - { - wrapper->Start(TInstant::Max()); - TAutoPtr<NActors::IEventHandle> handle; - auto* startEvent = runtime.GrabEdgeEvent<TProducerType::TCreateResponseEvent>(handle); - UNIT_ASSERT(startEvent != nullptr); - UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId); - UNIT_ASSERT_VALUES_EQUAL(startEvent->Response.Response.GetInit().GetMaxSeqNo(), 42); - } - - { - wrapper->Write(data); - TAutoPtr<NActors::IEventHandle> handle; - auto* writeEvent = runtime.GrabEdgeEvent<TProducerType::TCommitResponseEvent>(handle); - UNIT_ASSERT(writeEvent != nullptr); - UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId); - UNIT_ASSERT_VALUES_EQUAL(writeEvent->Response.Response.GetAck().GetSeqNo(), 100); - UNIT_ASSERT_EQUAL(writeEvent->Response.Data, data); - UNIT_ASSERT_VALUES_EQUAL(writeEvent->Response.SeqNo, 100); - } - - { - TError err; - err.SetDescription("trololo"); - deadPromise.SetValue(err); - - TAutoPtr<NActors::IEventHandle> handle; - auto* deadEvent = runtime.GrabEdgeEvent<TProducerType::TObjectIsDeadEvent>(handle); - UNIT_ASSERT(deadEvent != nullptr); - UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId); - UNIT_ASSERT_STRINGS_EQUAL(deadEvent->Response.GetDescription(), "trololo"); - } - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h deleted file mode 100644 index 630e8a35f89..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once -#include <library/cpp/actors/core/actor.h> -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/hfunc.h> - -namespace NPersQueue { - -// Fake object to hold actor id -class TFakeActor: public NActors::TActor<TFakeActor> { -public: - TFakeActor() - : TActor<TFakeActor>(&TFakeActor::StateFunc) - { - } - -private: - STFUNC(StateFunc) { - switch (ev->GetTypeRewrite()) { - CFunc(NActors::TEvents::TSystem::PoisonPill, Die); - } - } -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h deleted file mode 100644 index 4ad5fb0d48c..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once -#include "responses.h" -#include "actor_interface.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h> - -namespace NPersQueue { - -template <ui32 EventTypeId> -class TActorLogger - : public TActorHolder - , public ILogger -{ -public: - using TLogEvent = TEvPQLibLog<EventTypeId>; - -public: - explicit TActorLogger(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, int level) - : TActorHolder(actorSystem, parentActorID) - , Level(level) - { - } - - ~TActorLogger() override = default; - - void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) override { - if (!IsEnabled(level)) { - return; - } - ActorSystem->Send(ParentActorID, new TLogEvent( - msg, - sourceId, - sessionId, - static_cast<NActors::NLog::EPriority>(level) - )); - } - - bool IsEnabled(int level) const override { - return level <= Level; - } - -private: - int Level; -}; - -} - - - diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp deleted file mode 100644 index c663c3ac04a..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "actor_wrappers.h" -#include "persqueue.h" - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h> - -namespace NPersQueue { - -TPQLibActorsWrapper::TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, const TPQLibSettings& settings) - : PQLib(new TPQLib(settings)) - , ActorSystem(actorSystem) -{ -} - -TPQLibActorsWrapper::TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, std::shared_ptr<TPQLib> pqLib) - : PQLib(std::move(pqLib)) - , ActorSystem(actorSystem) -{ -} - -TPQLibActorsWrapper::~TPQLibActorsWrapper() = default; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h deleted file mode 100644 index 7a11416f4f7..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h +++ /dev/null @@ -1,87 +0,0 @@ -#pragma once -#include "actor_wrappers.h" - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -#include <library/cpp/actors/core/actorid.h> - -#include <util/generic/noncopyable.h> -#include <util/generic/ptr.h> - -namespace NActors { -class TActorSystem; -} // namespace NActors - -namespace NPersQueue { - -class TPQLibActorsWrapper : public TNonCopyable { -public: - explicit TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, const TPQLibSettings& settings = TPQLibSettings()); - TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, std::shared_ptr<TPQLib> pqLib); - ~TPQLibActorsWrapper(); - - // Producers creation - template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId> - THolder<TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>> CreateProducer( - const NActors::TActorId& parentActorID, - const TProducerSettings& settings, - TIntrusivePtr<ILogger> logger = nullptr, - bool deprecated = false - ) { - return new TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>( - ActorSystem, - parentActorID, - PQLib->CreateProducer(settings, std::move(logger), deprecated) - ); - } - - template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId> - THolder<TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>> CreateMultiClusterProducer( - const NActors::TActorId& parentActorID, - const TMultiClusterProducerSettings& settings, - TIntrusivePtr<ILogger> logger = nullptr, - bool deprecated = false - ) { - return new TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>( - ActorSystem, - parentActorID, - PQLib->CreateMultiClusterProducer(settings, std::move(logger), deprecated) - ); - } - - // Consumers creation - template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TGetMessageEventId> - THolder<TConsumerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TGetMessageEventId>> CreateConsumer( - const NActors::TActorId& parentActorID, - const TConsumerSettings& settings, - TIntrusivePtr<ILogger> logger = nullptr, - bool deprecated = false - ) { - return MakeHolder<TConsumerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TGetMessageEventId>>( - ActorSystem, - parentActorID, - PQLib->CreateConsumer(settings, std::move(logger),deprecated) - ); - } - - // Processors creation - template <ui32 TOriginDataEventId> - THolder<TProcessorActorWrapper<TOriginDataEventId>> CreateProcessor( - const NActors::TActorId& parentActorID, - const TProcessorSettings& settings, - TIntrusivePtr<ILogger> logger = nullptr, - bool deprecated = false - ) { - return new TProcessorActorWrapper<TOriginDataEventId>( - ActorSystem, - parentActorID, - PQLib->CreateProcessor(settings, std::move(logger), deprecated) - ); - } - -private: - std::shared_ptr<TPQLib> PQLib; - NActors::TActorSystem* ActorSystem; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h deleted file mode 100644 index 86e732d87cf..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> - -#include <library/cpp/actors/core/events.h> -#include <library/cpp/actors/core/event_local.h> - -namespace NPersQueue { - -template <class TResponseType_, ui32 EventTypeId> -struct TPQLibResponseEvent: public NActors::TEventLocal<TPQLibResponseEvent<TResponseType_, EventTypeId>, EventTypeId> { - using TResponseType = TResponseType_; - - explicit TPQLibResponseEvent(TResponseType&& response, ui64 requestId = 0) - : Response(std::move(response)) - , RequestId(requestId) - { - } - - TResponseType Response; - ui64 RequestId; -}; - -template <ui32 EventTypeId> -struct TEvPQLibLog : NActors::TEventLocal<TEvPQLibLog<EventTypeId>, EventTypeId> { - TEvPQLibLog(const TString& msg, const TString& sourceId, const TString& sessionId, NActors::NLog::EPriority level) - : Message(msg) - , SourceId(sourceId) - , SessionId(sessionId) - , Level(level) - {} - - TString Message; - TString SourceId; - TString SessionId; - NActors::NLog::EPriority Level; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp deleted file mode 100644 index b4d8431c8a8..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp +++ /dev/null @@ -1,113 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> - -#include <library/cpp/testing/unittest/registar.h> - -using namespace NThreading; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; - -namespace NPersQueue { -Y_UNIT_TEST_SUITE(TCompatibilityTest) { - void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { - const TProducerCommitResponse& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); - } - - void AssertWriteFailed(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { - const TProducerCommitResponse& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kError, "Msg: " << resp.Response); - } - - void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); - } - - void AssertReadContinuous(const TReadResponse::TData::TMessageBatch& batch, ui64 startSeqNo) { - for (ui32 i = 0; i < batch.MessageSize(); ++i) { - ui64 actualSeqNo = batch.GetMessage(i).GetMeta().GetSeqNo(); - UNIT_ASSERT_EQUAL_C(actualSeqNo, startSeqNo + i, "Wrong seqNo: " << actualSeqNo << ", expected: " << startSeqNo + i); - } - } - - void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response); - } - - void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response); - } - - void AssertLock(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kLock, "Msg: " << resp.Response); - } - - TProducerSettings MakeProducerSettings(const TTestServer& testServer) { - TProducerSettings producerSettings; - producerSettings.ReconnectOnFailure = true; - producerSettings.Topic = "topic1"; - producerSettings.SourceId = "123"; - producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - producerSettings.Codec = ECodec::LZOP; - producerSettings.ReconnectionDelay = TDuration::MilliSeconds(10); - return producerSettings; - } - - TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { - TConsumerSettings consumerSettings; - consumerSettings.ClientId = "user"; - consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - consumerSettings.Topics.push_back("topic1"); - return consumerSettings; - } - - Y_UNIT_TEST(ContinuesOperationsAfterPQLibDeath) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(); - - const size_t partitions = 1; - testServer.AnnoyingClient->FullInit(); - testServer.AnnoyingClient->CreateTopic("rt3.dc1--topic1", partitions); - - testServer.WaitInit("topic1"); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLibSettings pqLibSettings; - pqLibSettings.DefaultLogger = logger; - THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); - - auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, true); - UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); - auto isProducerDead = producer->IsDead(); - UNIT_ASSERT(!isProducerDead.HasValue()); - - auto consumer = PQLib->CreateConsumer(MakeConsumerSettings(testServer), logger, true); - - PQLib = nullptr; - - UNIT_ASSERT(!consumer->Start().GetValueSync().Response.HasError()); - auto isConsumerDead = consumer->IsDead(); - UNIT_ASSERT(!isConsumerDead.HasValue()); - - auto write1 = producer->Write(1, TString("blob1")); - auto write2 = producer->Write(2, TString("blob2")); - - UNIT_ASSERT(!write1.GetValueSync().Response.HasError()); - UNIT_ASSERT(!write2.GetValueSync().Response.HasError()); - - UNIT_ASSERT(!consumer->GetNextMessage().GetValueSync().Response.HasError()); - - producer = nullptr; - consumer = nullptr; - - isProducerDead.GetValueSync(); - isConsumerDead.GetValueSync(); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h deleted file mode 100644 index 48df7b0cdd4..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include "logger.h" - -#include <kikimr/yndx/api/protos/persqueue.pb.h> - -#include <library/cpp/tvmauth/client/facade.h> - -namespace NPersQueue { - -const TString& GetTvmPqServiceName(); - -class ICredentialsProvider { -public: - virtual ~ICredentialsProvider() = default; - virtual void FillAuthInfo(NPersQueueCommon::TCredentials* authInfo) const = 0; -}; - - -class ITVMCredentialsForwarder : public ICredentialsProvider { -public: - virtual ~ITVMCredentialsForwarder() = default; - virtual void SetTVMServiceTicket(const TString& ticket) = 0; -}; - -std::shared_ptr<ICredentialsProvider> CreateInsecureCredentialsProvider(); - -/* - * Builder for direct usage of TVM. For Qloud environment use CreateTVMQloudCredentialsProvider - */ -std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, TIntrusivePtr<ILogger> logger = nullptr); -std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, const TString& dstAlias = GetTvmPqServiceName()); - -/* - * forward TVM service ticket with ITMVCredentialsProvider method SetTVMServiceTicket ; threadsafe - */ -std::shared_ptr<ITVMCredentialsForwarder> CreateTVMCredentialsForwarder(); - -/* - * tvmClient settings must contain LogBroker client id under an alias - */ -std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, TIntrusivePtr<ILogger> logger = nullptr, const TString& dstAlias = GetTvmPqServiceName()); - -std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, TIntrusivePtr<ILogger> logger = nullptr, const TDuration refreshPeriod = TDuration::Minutes(10), ui32 port = 1); -std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, TIntrusivePtr<ILogger> logger = nullptr, const TDuration refreshPeriod = TDuration::Minutes(10), ui32 port = 1); -std::shared_ptr<ICredentialsProvider> CreateOAuthCredentialsProvider(const TString& token); - -/* - * Provider that takes IAM ticket from metadata service. - */ -std::shared_ptr<ICredentialsProvider> CreateIAMCredentialsForwarder(TIntrusivePtr<ILogger> logger); - -/* - * Provider that makes IAM ticket from JWT token. - */ -std::shared_ptr<ICredentialsProvider> CreateIAMJwtFileCredentialsForwarder(const TString& jwtKeyFilename, TIntrusivePtr<ILogger> logger, - const TString& endpoint = "iam.api.cloud.yandex.net", const TDuration& refreshPeriod = TDuration::Hours(1), - const TDuration& requestTimeout = TDuration::Seconds(10)); - -std::shared_ptr<ICredentialsProvider> CreateIAMJwtParamsCredentialsForwarder(const TString& jwtParams, TIntrusivePtr<ILogger> logger, - const TString& endpoint = "iam.api.cloud.yandex.net", const TDuration& refreshPeriod = TDuration::Hours(1), - const TDuration& requestTimeout = TDuration::Seconds(10)); -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h deleted file mode 100644 index 975b2ccd729..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include "types.h" -#include "responses.h" - -#include <kikimr/yndx/api/protos/persqueue.pb.h> - -#include <library/cpp/threading/future/future.h> - -#include <util/generic/vector.h> - -namespace NPersQueue { - -class IConsumer { -public: - virtual ~IConsumer() = default; - - // Start consumer. - // Consumer can be used after its start will be finished. - virtual NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept = 0; - - NThreading::TFuture<TConsumerCreateResponse> Start(TDuration timeout) noexcept { - return Start(TInstant::Now() + timeout); - } - - //read result according to settings or Lock/Release requests from server(if UseLockSession is true) - virtual NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept = 0; - - //commit processed reads - virtual void Commit(const TVector<ui64>& cookies) noexcept = 0; - - //will request status - response will be received in GetNextMessage with EMT_STATUS - virtual void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { Y_UNUSED(topic); Y_UNUSED(partition); Y_UNUSED(generation); }; - - //will be signalled in case of errors - you may use this future if you can't use GetNextMessage() //out of memory or any other reason - virtual NThreading::TFuture<TError> IsDead() noexcept = 0; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp deleted file mode 100644 index c29b1d8dd81..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp +++ /dev/null @@ -1,421 +0,0 @@ -#include "channel.h" -#include "channel_p.h" -#include "internals.h" -#include "persqueue_p.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> - -#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - -#include <util/random/random.h> -#include <util/string/builder.h> - -#include <grpc++/create_channel.h> - -const ui32 MAX_MESSAGE_SIZE = 150 * 1024 * 1024; -const TDuration KEEP_ALIVE_TIME = TDuration::Seconds(90); -const TDuration KEEP_ALIVE_TIMEOUT = TDuration::Seconds(1); -constexpr ui64 MAGIC_COOKIE_VALUE = 123456789; - -namespace NPersQueue { - -void FillMetaHeaders(grpc::ClientContext& context, const TString& database, ICredentialsProvider* credentials) { - context.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, GetToken(credentials)); - context.AddMetadata(NYdb::YDB_DATABASE_HEADER, database); -} - -TServerSetting ApplyClusterEndpoint(const TServerSetting& server, const TString& endpoint) { - TServerSetting ret = server; - ret.UseLogbrokerCDS = EClusterDiscoveryUsageMode::DontUse; - TStringBuf endpointWithPort = endpoint; - TStringBuf host, port; - ui16 parsedPort = 0; - if (endpointWithPort.TryRSplit(':', host, port) && TryFromString(port, parsedPort)) { - ret.Address = TString(host); - ret.Port = parsedPort; - } else { - ret.Address = endpoint; - } - return ret; -} - -bool UseCDS(const TServerSetting& server) { - switch (server.UseLogbrokerCDS) { - case EClusterDiscoveryUsageMode::Auto: - return server.Address == "logbroker.yandex.net"sv || server.Address == "logbroker-prestable.yandex.net"sv; - case EClusterDiscoveryUsageMode::Use: - return true; - case EClusterDiscoveryUsageMode::DontUse: - return false; - } -} - -class TChannelImpl::TGetProxyHandler : public IHandler { -public: - TGetProxyHandler(TChannelImplPtr ptr) - : Ptr(ptr) - {} - - void Done() override { - Ptr->OnGetProxyDone(); - } - - void Destroy(const TError& reason) override { - Ptr->Destroy(reason); - } - - TString ToString() override { return "GetProxyHandler"; } - -protected: - TChannelImplPtr Ptr; -}; - -class TChannelOverCdsBaseImpl::TCdsResponseHandler : public IHandler { -public: - TCdsResponseHandler(TChannelOverCdsBaseImplPtr ptr) - : Ptr(ptr) - {} - - void Done() override { - Ptr->OnCdsRequestDone(); - } - - void Destroy(const TError& reason) override { - Ptr->Destroy(reason); - } - - TString ToString() override { return "CDS-ResponseHandler"; } - -protected: - TChannelOverCdsBaseImplPtr Ptr; -}; - -TChannel::TChannel( - const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy -) { - MakeImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy); -} - -TChannel::TChannel( - const TProducerSettings& settings, TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy -) { - if (UseCDS(settings.Server)) - Impl = new TProducerChannelOverCdsImpl(settings, pqLib, std::move(logger), preferLocalProxy); - else - MakeImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, preferLocalProxy); -} - -void TChannel::MakeImpl( - const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, - TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy -) { - Impl = MakeIntrusive<TChannelOverDiscoveryImpl>( - server, credentialsProvider, pqLib, std::move(logger), preferLocalProxy - ); -} - -NThreading::TFuture<TChannelInfo> TChannel::GetChannel() { - return Impl->GetChannel(); -} - -TChannel::~TChannel() { - Impl->TryCancel(); - Impl->Wait(); -} - -void TChannel::Start() { - Impl->Start(); -} - -TChannelImpl::TChannelImpl(const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy) - : Promise(NThreading::NewPromise<TChannelInfo>()) - , Server(server) - , SelectedEndpoint(Server.GetFullAddressString()) - , CredentialsProvider(credentialsProvider) - , PreferLocalProxy(preferLocalProxy) - , Logger(std::move(logger)) - , PQLib(pqLib) - , CQ(pqLib->GetCompletionQueue()) - , ChooseProxyFinished(0) -{} - -TChannelOverDiscoveryImpl::TChannelOverDiscoveryImpl( - const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy -) - : TChannelImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy) -{ - CreationTimeout = PQLib->GetSettings().ChannelCreationTimeout; -} - -TChannelOverCdsBaseImpl::TChannelOverCdsBaseImpl(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy -) - : TChannelOverDiscoveryImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy) - , CdsRequestFinished(0) -{} - -void TChannelOverDiscoveryImpl::Start() { - Channel = CreateGrpcChannel(SelectedEndpoint); - Stub = Ydb::Discovery::V1::DiscoveryService::NewStub(Channel); - IHandlerPtr handler = new TGetProxyHandler(this); - Context.set_deadline(TInstant::Now() + CreationTimeout); - Ydb::Discovery::ListEndpointsRequest request; - request.set_database(Server.Database); - FillMetaHeaders(Context, Server.Database, CredentialsProvider.get()); - DEBUG_LOG("Send list endpoints request: " << request, "", ""); - Rpc = Stub->AsyncListEndpoints(&Context, request, CQ.get()); - Rpc->Finish(&Response, &Status, new TQueueEvent(std::move(handler))); -} - -void TChannelOverCdsBaseImpl::Start() { - Channel = CreateGrpcChannel(Server.GetFullAddressString()); - Stub = Ydb::PersQueue::V1::ClusterDiscoveryService::NewStub(Channel); - IHandlerPtr handler = new TCdsResponseHandler(this); - CdsContext.set_deadline(TInstant::Now() + PQLib->GetSettings().ChannelCreationTimeout); - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request = GetCdsRequest(); - FillMetaHeaders(CdsContext, Server.Database, CredentialsProvider.get()); - DEBUG_LOG("Send cds request: " << request, "", ""); - CreationStartTime = TInstant::Now(); - Rpc = Stub->AsyncDiscoverClusters(&CdsContext, request, CQ.get()); - Rpc->Finish(&Response, &Status, new TQueueEvent(std::move(handler))); -} - -NThreading::TFuture<TChannelInfo> TChannelImpl::GetChannel() { - return Promise.GetFuture(); -} - -bool TChannelImpl::TryCancel() { - if (AtomicSwap(&ChooseProxyFinished, 1) == 0) { - Y_ASSERT(!Promise.HasValue()); - Promise.SetValue(TChannelInfo{nullptr, 0}); - Context.TryCancel(); - return true; - } - return false; -} - -TChannelImpl::~TChannelImpl() = default; - -std::shared_ptr<grpc::Channel> TChannelImpl::CreateGrpcChannel(const TString& address) { - grpc::ChannelArguments args; - args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, KEEP_ALIVE_TIME.MilliSeconds()); - args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, KEEP_ALIVE_TIMEOUT.MilliSeconds()); - args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); - args.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0); - args.SetInt(GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, KEEP_ALIVE_TIME.MilliSeconds()); - - args.SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, MAX_MESSAGE_SIZE); - args.SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, MAX_MESSAGE_SIZE); - - //TODO: fill there other settings from PQLib - DEBUG_LOG("Creating grpc channel to \"" << address << "\"", "", ""); - if (Server.UseSecureConnection) - return grpc::CreateCustomChannel(address, - grpc::SslCredentials(grpc::SslCredentialsOptions{Server.CaCert, "", ""}), - args); - else - return grpc::CreateCustomChannel(address, grpc::InsecureChannelCredentials(), args); -} - -TString TChannelImpl::GetProxyAddress(const TString& proxyName, ui32 port) { - TStringBuilder address; - if (proxyName.find(':') != TString::npos && proxyName.find('[') == TString::npos) { //fix for ipv6 adresses - address << "[" << proxyName << "]"; - } else { - address << proxyName; - } - if (port) - address << ":" << port; - else - address << ":" << Server.Port; - return std::move(address); -} - -void TChannelImpl::OnGetProxyDone() { - if (AtomicSwap(&ChooseProxyFinished, 1) == 0) { - const bool ok = Status.ok(); - Y_ASSERT(!Promise.HasValue()); - if (ok) { - ProcessGetProxyResponse(); - } else { - if (Status.error_code() == grpc::CANCELLED) { - INFO_LOG("Grpc request is canceled by user", "", ""); - } else { - const auto& msg = Status.error_message(); - TString res(msg.data(), msg.length()); - ERR_LOG("Grpc error " << static_cast<int>(Status.error_code()) << ": \"" << res << "\"", "", ""); - } - StartFailed(); - } - } - SetDone(); -} - -void TChannelImpl::StartFailed() { - AtomicSet(ChooseProxyFinished, 1); - Promise.SetValue(TChannelInfo{nullptr, 0}); -} - -void TChannelOverDiscoveryImpl::ProcessGetProxyResponse() { - Ydb::Discovery::ListEndpointsResult result; - auto& reply = Response.operation().result(); - reply.UnpackTo(&result); - - auto has_service = [](const Ydb::Discovery::EndpointInfo& endpoint) { - for (auto& service: endpoint.service()) { - if (service == "pq") - return true; - } - return false; - }; - TVector<std::pair<TString, ui32>> validEps; - for (const auto& ep : result.endpoints()) { - if (has_service(ep) && (!Server.UseSecureConnection || ep.ssl())) { - validEps.emplace_back(ep.address(), ep.port()); - } - } - if (validEps.empty()) { - ERR_LOG("No valid endpoints to connect!", "", ""); - StartFailed(); - } else { - auto& selected = validEps[RandomNumber<ui32>(validEps.size())]; - DEBUG_LOG("Selected endpoint {\"" << selected.first << "\", " << selected.second << "}", "", ""); - auto channel = CreateGrpcChannel(GetProxyAddress(selected.first, selected.second)); - Promise.SetValue(TChannelInfo{std::move(channel), MAGIC_COOKIE_VALUE}); - } -} - -void TChannelOverCdsBaseImpl::OnCdsRequestDone() { - DEBUG_LOG("ON CDS request done: " << Response, "", ""); - - if (AtomicSwap(&CdsRequestFinished, 1) == 0) { - const bool ok = Status.ok(); - if (ok) { - if (ProcessCdsResponse()) { - CreationTimeout = CreationTimeout - (TInstant::Now() - CreationStartTime); - TChannelOverDiscoveryImpl::Start(); - // Don't set AllDone event, because we are about to continue node discovery - // via TChannelOverDiscoveryImpl part of implementation. - return; - } else { - StartFailed(); - } - } else { - if (Status.error_code() == grpc::CANCELLED) { - INFO_LOG("Grpc request is canceled by user", "", ""); - } else { - const auto& msg = Status.error_message(); - TString res(msg.data(), msg.length()); - ERR_LOG("Grpc error " << static_cast<int>(Status.error_code()) << ": \"" << res << "\"", "", ""); - } - StartFailed(); - } - } - SetDone(); -} - -bool TChannelOverCdsBaseImpl::TryCancel() { - if (AtomicSwap(&CdsRequestFinished, 1) == 0) { - Y_ASSERT(!Promise.HasValue()); - Promise.SetValue(TChannelInfo{nullptr, 0}); - CdsContext.TryCancel(); - return true; - } else { - TChannelOverDiscoveryImpl::TryCancel(); - } - return false; -} - -TProducerChannelOverCdsImpl::TProducerChannelOverCdsImpl(const TProducerSettings& settings, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy) - : TChannelOverCdsBaseImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, preferLocalProxy) - , Topic(settings.Topic) - , SourceId(settings.SourceId) - , PreferredCluster(settings.PreferredCluster) -{ -} - -Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TProducerChannelOverCdsImpl::GetCdsRequest() const { - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; - auto* params = request.add_write_sessions(); - params->set_topic(Topic); - params->set_source_id(SourceId); - params->set_preferred_cluster_name(PreferredCluster); // Can be empty. - return request; -} - -bool TProducerChannelOverCdsImpl::ProcessCdsResponse() { - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result; - auto& reply = Response.operation().result(); - reply.UnpackTo(&result); - DEBUG_LOG("Process CDS result: " << result, "", ""); - for (auto& session: result.write_sessions_clusters()) { - for (auto& cluster: session.clusters()) { - if (cluster.available()) { - const TServerSetting selectedServer = ApplyClusterEndpoint(Server, cluster.endpoint()); - SelectedEndpoint = GetProxyAddress(selectedServer.Address, selectedServer.Port); - return true; - } - } - } - ERR_LOG("Could not find valid cluster in CDS response", "", ""); - return false; -} - -TConsumerChannelOverCdsImpl::TConsumerChannelOverCdsImpl(const TConsumerSettings& settings, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger) - : TChannelOverCdsBaseImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, true) - , Result(std::make_shared<TResult>()) - , Topics(settings.Topics) -{ -} - -Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TConsumerChannelOverCdsImpl::GetCdsRequest() const { - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; - for (const TString& topic : Topics) { - auto* params = request.add_read_sessions(); - params->set_topic(topic); - params->mutable_all_original(); // set all_original - } - return request; -} - -bool TConsumerChannelOverCdsImpl::ProcessCdsResponse() { - auto& reply = Response.operation().result(); - reply.UnpackTo(&Result->second); - Result->first = Status; - DEBUG_LOG("Got CDS result: " << Result->second, "", ""); - return false; // Signals future -} - -void TConsumerChannelOverCdsImpl::StartFailed() { - Result->first = Status; - TChannelOverCdsBaseImpl::StartFailed(); -} - -void TChannelImpl::SetDone() { - AllDoneEvent.Signal(); -} - -void TChannelImpl::Destroy(const TError& error) { - if (AtomicSwap(&ChooseProxyFinished, 1) == 0) { - const auto& msg = Status.error_message(); - TString res(msg.data(), msg.length()); - - ERR_LOG("Got proxy error response " << error << " grpc error " << res, "", ""); - //TODO: add here status from Status to response, log status - - Y_ASSERT(!Promise.HasValue()); - Promise.SetValue(TChannelInfo{nullptr, 0}); - } - - SetDone(); -} - -void TChannelImpl::Wait() { - AllDoneEvent.Wait(); -} - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h deleted file mode 100644 index 630b0976758..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> - -#include <library/cpp/threading/future/future.h> - -#include <deque> - -namespace NPersQueue { - -struct TChannelInfo { - std::shared_ptr<grpc::Channel> Channel; - ui64 ProxyCookie; -}; - -class TChannel; -using TChannelPtr = TIntrusivePtr<TChannel>; - -class TPQLibPrivate; - -struct TChannelHolder { - TChannelPtr ChannelPtr; - NThreading::TFuture<TChannelInfo> ChannelInfo; -}; - -class TChannelImpl; -using TChannelImplPtr = TIntrusivePtr<TChannelImpl>; - -class TChannel: public TAtomicRefCount<TChannel> { -public: - friend class TPQLibPrivate; - - NThreading::TFuture<TChannelInfo> GetChannel(); - - ~TChannel(); - - void Start(); - -private: - TChannel(const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false); - TChannel(const TProducerSettings& settings, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false); - void MakeImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false); - TChannelImplPtr Impl; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h deleted file mode 100644 index 10f18abb047..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h +++ /dev/null @@ -1,156 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include "channel.h" -#include "scheduler.h" -#include "internals.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include <library/cpp/threading/future/future.h> -#include <ydb/public/api/protos/ydb_discovery.pb.h> -#include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h> -#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> - -#include <deque> - -namespace NPersQueue { - -class TPQLibPrivate; - -// Apply endpoint that was taken from CDS. -TServerSetting ApplyClusterEndpoint(const TServerSetting& server, const TString& endpoint); - -class TChannelImpl : public TAtomicRefCount<TChannelImpl> { -public: - NThreading::TFuture<TChannelInfo> GetChannel(); - - virtual ~TChannelImpl(); - - virtual bool TryCancel(); - - TChannelImpl(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy); - - virtual void Start() = 0; - - void Wait(); - -protected: - friend class TPQLibPrivate; - class TGetProxyHandler; - - void Destroy(const TError& error); - virtual void StartFailed(); - void SetDone(); - void OnGetProxyDone(); - virtual void ProcessGetProxyResponse() = 0; - std::shared_ptr<grpc::Channel> CreateGrpcChannel(const TString& address); - TString GetProxyAddress(const TString& proxyName, ui32 port = 0); - TString GetToken(); - - NThreading::TPromise<TChannelInfo> Promise; - TServerSetting Server; - TString SelectedEndpoint; - std::shared_ptr<ICredentialsProvider> CredentialsProvider; - bool PreferLocalProxy; - - TIntrusivePtr<ILogger> Logger; - TPQLibPrivate* PQLib; - -protected: - std::shared_ptr<grpc::CompletionQueue> CQ; - std::shared_ptr<grpc::Channel> Channel; - grpc::ClientContext Context; - grpc::Status Status; - - TAutoEvent AllDoneEvent; - - TAtomic ChooseProxyFinished; -}; - -class TChannelOverDiscoveryImpl : public TChannelImpl { -public: - TChannelOverDiscoveryImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy); - void Start() override; - -protected: - void ProcessGetProxyResponse() override; - TDuration CreationTimeout; - -private: - std::unique_ptr<Ydb::Discovery::V1::DiscoveryService::Stub> Stub; - std::unique_ptr<grpc::ClientAsyncResponseReader<Ydb::Discovery::ListEndpointsResponse>> Rpc; - Ydb::Discovery::ListEndpointsResponse Response; -}; - - -class TChannelOverCdsBaseImpl : public TChannelOverDiscoveryImpl { -public: - TChannelOverCdsBaseImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy); - void Start() override; - void OnCdsRequestDone(); - - bool TryCancel() override; - -private: - virtual Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const = 0; - virtual bool ProcessCdsResponse() = 0; - -protected: - class TCdsResponseHandler; - -private: - std::unique_ptr<Ydb::PersQueue::V1::ClusterDiscoveryService::Stub> Stub; - std::unique_ptr<grpc::ClientAsyncResponseReader<Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse>> Rpc; - -protected: - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse Response; - -private: - grpc::ClientContext CdsContext; - - TInstant CreationStartTime = TInstant::Zero(); - TAtomic CdsRequestFinished; -}; -using TChannelOverCdsBaseImplPtr = TIntrusivePtr<TChannelOverCdsBaseImpl>; - -class TProducerChannelOverCdsImpl : public TChannelOverCdsBaseImpl { -public: - TProducerChannelOverCdsImpl(const TProducerSettings& settings, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy); - -private: - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const override; - bool ProcessCdsResponse() override; - -private: - TString Topic; - TString SourceId; - TString PreferredCluster; -}; - -class TConsumerChannelOverCdsImpl : public TChannelOverCdsBaseImpl { -public: - using TResult = std::pair<grpc::Status, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult>; - using TResultPtr = std::shared_ptr<TResult>; - -public: - TConsumerChannelOverCdsImpl(const TConsumerSettings& settings, TPQLibPrivate* pqLib, - TIntrusivePtr<ILogger> logger); - - TResultPtr GetResultPtr() const { - return Result; - } - -private: - Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const override; - bool ProcessCdsResponse() override; - void StartFailed() override; - -private: - TResultPtr Result; - TVector<TString> Topics; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp deleted file mode 100644 index 14c2871a54c..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp +++ /dev/null @@ -1,256 +0,0 @@ - -#include "compat_producer.h" -#include "persqueue_p.h" - -namespace NPersQueue { - - using namespace NYdb::NPersQueue; - - - static TWriteSessionSettings ConvertToWriteSessionSettings(const TProducerSettings& ps) { - TWriteSessionSettings settings; - settings.Path(ps.Topic) - .MessageGroupId(ps.SourceId) - .RetryPolicy(ps.ReconnectOnFailure ? NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy(ps.ReconnectionDelay, ps.ReconnectionDelay, ps.MaxReconnectionDelay, - ps.MaxAttempts, TDuration::Max(), 2.0, [](NYdb::EStatus){ return ERetryErrorClass::ShortRetry; }) - : NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()) - .Codec(NYdb::NPersQueue::ECodec(ps.Codec + 1)) - .CompressionLevel(ps.Quality) - .ValidateSeqNo(false); - for (auto& attr : ps.ExtraAttrs) { - settings.AppendSessionMeta(attr.first, attr.second); - } - - if (ps.PreferredCluster) { - settings.PreferredCluster(ps.PreferredCluster); - } - if (ps.DisableCDS || ps.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::DontUse) { - settings.ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Off); - } else if (ps.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::Use) { - settings.ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::On); - } - - return settings; - } - - TYdbSdkCompatibilityProducer::~TYdbSdkCompatibilityProducer() { - WriteSession->Close(TDuration::Seconds(0)); - - NotifyClient(NErrorCode::OK, "producer object destroyed by client"); - } - - TYdbSdkCompatibilityProducer::TYdbSdkCompatibilityProducer(const TProducerSettings& settings, TPersQueueClient& persQueueClient, - std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) - : IProducerImpl(destroyEventRef, pqLib) - { - IsDeadPromise = ::NThreading::NewPromise<TError>(); - TWriteSessionSettings sessionSettings = ConvertToWriteSessionSettings(settings); - Closed = false; - WriteSession = persQueueClient.CreateWriteSession(sessionSettings); - } - - void TYdbSdkCompatibilityProducer::SubscribeToNextEvent() { - NextEvent = WriteSession->WaitEvent(); - std::weak_ptr<TYdbSdkCompatibilityProducer> self = shared_from_this(); - NextEvent.Subscribe([self](const auto& future) { - auto selfShared = self.lock(); - Y_UNUSED(future); - if (selfShared) { - selfShared->DoProcessNextEvent(); - } - }); - } - - void TYdbSdkCompatibilityProducer::Init() noexcept { - } - - NThreading::TFuture<TProducerCreateResponse> TYdbSdkCompatibilityProducer::Start(TInstant deadline) noexcept { - Y_UNUSED(deadline); // TODO: start timeout? - SubscribeToNextEvent(); - return WriteSession->GetInitSeqNo().Apply([](const auto& future) { - TWriteResponse response; - if (future.HasException()) { - response.MutableError()->SetDescription("session closed"); - response.MutableError()->SetCode(NErrorCode::ERROR); - } else { - response.MutableInit()->SetMaxSeqNo(future.GetValue()); - } - TProducerCreateResponse res(std::move(response)); - - return NThreading::MakeFuture<TProducerCreateResponse>(res); - }); - } - - void TYdbSdkCompatibilityProducer::WriteImpl(NThreading::TPromise<TProducerCommitResponse>& promise, TMaybe<TProducerSeqNo> seqNo, ::NPersQueue::TData data) noexcept { - TContinuationToken *contToken = nullptr; - bool write = false; - with_lock(Lock) { - if (!Closed) { - contToken = ContToken.Get(); - if(contToken == nullptr || !ToWrite.empty()) { - ToWrite.emplace(promise, seqNo, data); - return; - } else { // contToken here and ToWrite is empty - can write right now - ToAck.emplace(promise, seqNo, data); - write = true; - } - } - } - if (write) { - if (data.IsEncoded()) { - WriteSession->WriteEncoded(std::move(*contToken), data.GetEncodedData(), NYdb::NPersQueue::ECodec(data.GetCodecType() + 1), data.GetOriginalSize(), seqNo); // can trigger NextEvent.Subscribe() in the same thread - } else { - WriteSession->Write(std::move(*contToken), data.GetSourceData(), seqNo); // can trigger NextEvent.Subscribe() in the same thread - } - } else { - TWriteResponse wr; - wr.MutableError()->SetCode(NErrorCode::ERROR); - wr.MutableError()->SetDescription("session closed"); - TProducerCommitResponse resp(seqNo ? *seqNo : 0, data, std::move(wr)); - promise.SetValue(resp); - } - } - - void TYdbSdkCompatibilityProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, ::NPersQueue::TData data) noexcept { - return WriteImpl(promise, {}, std::move(data)); // similar to TCompressingProducer - } - - void TYdbSdkCompatibilityProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, ::NPersQueue::TData data) noexcept { - return WriteImpl(promise, seqNo, std::move(data)); // similar to TCompressingProducer - } - - void TYdbSdkCompatibilityProducer::Cancel() { - WriteSession->Close(TDuration::Seconds(0)); - } - - NErrorCode::EErrorCode GetErrorCode(const NYdb::EStatus status) { - switch(status) { - case NYdb::EStatus::SUCCESS: - return NErrorCode::OK; - case NYdb::EStatus::UNAVAILABLE: - return NErrorCode::INITIALIZING; - case NYdb::EStatus::OVERLOADED: - return NErrorCode::OVERLOAD; - case NYdb::EStatus::BAD_REQUEST: - return NErrorCode::BAD_REQUEST; - case NYdb::EStatus::NOT_FOUND: - case NYdb::EStatus::SCHEME_ERROR: - return NErrorCode::UNKNOWN_TOPIC; - case NYdb::EStatus::UNSUPPORTED: - return NErrorCode::BAD_REQUEST; - case NYdb::EStatus::UNAUTHORIZED: - return NErrorCode::ACCESS_DENIED; - default: - return NErrorCode::ERROR; - } - return NErrorCode::ERROR; - } - - struct TToNotify { - NThreading::TPromise<TProducerCommitResponse> Promise; - TProducerCommitResponse Resp; - - TToNotify(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerCommitResponse&& resp) - : Promise(promise) - , Resp(resp) - {} - }; - - - void TYdbSdkCompatibilityProducer::NotifyClient(NErrorCode::EErrorCode code, const TString& reason) { - - TError err; - err.SetCode(code); - err.SetDescription(reason); - std::queue<TToNotify> toNotify; - with_lock(Lock) { - if (Closed) return; - Closed = true; - while(!ToAck.empty()) { - TMsgData& item = ToAck.front(); - TWriteResponse wr; - wr.MutableError()->CopyFrom(err); - TProducerCommitResponse resp(item.SeqNo ? *item.SeqNo : 0, item.Data, std::move(wr)); - toNotify.emplace(item.Promise, std::move(resp)); - ToAck.pop(); - } - while(!ToWrite.empty()) { - TMsgData& item = ToWrite.front(); - TWriteResponse wr; - wr.MutableError()->CopyFrom(err); - TProducerCommitResponse resp(item.SeqNo ? *item.SeqNo : 0, item.Data, std::move(wr)); - toNotify.emplace(item.Promise, std::move(resp)); - ToWrite.pop(); - } - } - while(!toNotify.empty()) { - toNotify.front().Promise.SetValue(toNotify.front().Resp); - toNotify.pop(); - } - IsDeadPromise.SetValue(err); - DestroyPQLibRef(); - } - - void TYdbSdkCompatibilityProducer::DoProcessNextEvent() { - - std::queue<TToNotify> toNotify; - - TVector<TWriteSessionEvent::TEvent> events = WriteSession->GetEvents(false); - //Y_VERIFY(!events.empty()); - for (auto& event : events) { - if(std::holds_alternative<TSessionClosedEvent>(event)) { - NotifyClient(GetErrorCode(std::get<TSessionClosedEvent>(event).GetStatus()), std::get<TSessionClosedEvent>(event).DebugString() ); - return; - } else if(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)) { - - // get seqNos, signal their promises - TWriteSessionEvent::TAcksEvent& acksEvent = std::get<TWriteSessionEvent::TAcksEvent>(event); - std::queue<TToNotify> toNotify; - for(TWriteSessionEvent::TWriteAck& ackEvent : acksEvent.Acks) { - TWriteResponse writeResp; - auto ackMsg = writeResp.MutableAck(); - ackMsg->SetSeqNo(ackEvent.SeqNo); - ackMsg->SetAlreadyWritten(ackEvent.State == TWriteSessionEvent::TWriteAck::EEventState::EES_ALREADY_WRITTEN); - ackMsg->SetOffset(ackEvent.Details ? ackEvent.Details->Offset : 0); - with_lock(Lock) { - Y_ASSERT(!ToAck.empty()); - TProducerCommitResponse resp(ackEvent.SeqNo, ToAck.front().Data, std::move(writeResp)); - toNotify.emplace(ToAck.front().Promise, std::move(resp)); - ToAck.pop(); - } - } - while(!toNotify.empty()) { - toNotify.front().Promise.SetValue(toNotify.front().Resp); - toNotify.pop(); - } - } else if(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) { - TWriteSessionEvent::TReadyToAcceptEvent& readyEvent = std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event); - TContinuationToken contToken = std::move(readyEvent.ContinuationToken); // IF there is only one only movable contToken THEN this is thread safe - TString strData; - TMaybe<TProducerSeqNo> seqNo; - bool write = false; - with_lock(Lock) { - if(ToWrite.empty()) { - ContToken = std::move(contToken); - } else { - strData = ToWrite.front().Data.GetSourceData(); - seqNo = ToWrite.front().SeqNo; - ToAck.push(ToWrite.front()); - ToWrite.pop(); - write = true; - } - } - if (write) { - WriteSession->Write(std::move(contToken), strData, seqNo); - } - } - } - - SubscribeToNextEvent(); - } - - NThreading::TFuture<TError> TYdbSdkCompatibilityProducer::IsDead() noexcept { - return IsDeadPromise.GetFuture(); - } - -} // namespace NYdb::NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h deleted file mode 100644 index e5287d46944..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once - -#include "iproducer_p.h" - -#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h> - -#include <memory> -#include <queue> - -namespace NPersQueue { - - -/** - * Compatibility producer: - * wraps old "interface" around the new one - */ - -class TYdbSdkCompatibilityProducer: public IProducerImpl, public std::enable_shared_from_this<TYdbSdkCompatibilityProducer> { - - struct TMsgData { - NThreading::TPromise<TProducerCommitResponse> Promise; - TMaybe<TProducerSeqNo> SeqNo; - TData Data; - - TMsgData() {}; - TMsgData(NThreading::TPromise<TProducerCommitResponse> promise, TMaybe<TProducerSeqNo> seqNo, TData data) - : Promise(promise) - , SeqNo(seqNo) - , Data(data) {} - }; - - std::shared_ptr<NYdb::NPersQueue::IWriteSession> WriteSession; - NThreading::TPromise<TError> IsDeadPromise; - NThreading::TFuture<void> NextEvent; - - TMaybe<NYdb::NPersQueue::TContinuationToken> ContToken; - std::queue<TMsgData> ToWrite; - std::queue<TMsgData> ToAck; - TSpinLock Lock; - bool Closed; - - void DoProcessNextEvent(); - void SubscribeToNextEvent(); - - void WriteImpl(NThreading::TPromise<TProducerCommitResponse>& promise, TMaybe<TProducerSeqNo> seqNo, TData data) noexcept; - - void NotifyClient(NErrorCode::EErrorCode code, const TString& reason); - -public: - using IProducerImpl::Write; - NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept override; - NThreading::TFuture<TError> IsDead() noexcept override; - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override; - void Cancel() override; - void Init() noexcept override; - - ~TYdbSdkCompatibilityProducer(); - - TYdbSdkCompatibilityProducer(const TProducerSettings& settings, NYdb::NPersQueue::TPersQueueClient& persQueueClient, - std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); - -}; - - -} // namespace NYdb::NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp deleted file mode 100644 index ecb6f70944d..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp +++ /dev/null @@ -1,155 +0,0 @@ -#include "compressing_producer.h" -#include "persqueue_p.h" - -namespace NPersQueue { - -TCompressingProducer::TCompressingProducer(std::shared_ptr<IProducerImpl> subproducer, ECodec defaultCodec, int quality, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) - : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) - , Logger(std::move(logger)) - , Subproducer(std::move(subproducer)) - , DefaultCodec(defaultCodec) - , Quality(quality) -{ - if (defaultCodec == ECodec::DEFAULT) { - ythrow yexception() << "Producer codec can't be ECodec::DEFAULT"; - } -} - -TCompressingProducer::~TCompressingProducer() { - DestroyQueue("Destructor called"); -} - -NThreading::TFuture<TProducerCreateResponse> TCompressingProducer::Start(TInstant deadline) noexcept { - return Subproducer->Start(deadline); -} - -NThreading::TFuture<TData> TCompressingProducer::Enqueue(TData data) { - NThreading::TPromise<TData> promise = NThreading::NewPromise<TData>(); - std::weak_ptr<TCompressingProducer> self = shared_from_this(); - const void* queueTag = this; - const ECodec defaultCodec = DefaultCodec; - const int quality = Quality; - auto compress = [promise, self, data, queueTag, defaultCodec, quality, pqLib = PQLib.Get()]() mutable { - Compress(promise, std::move(data), defaultCodec, quality, queueTag, self, pqLib); - }; - Y_VERIFY(PQLib->GetCompressionPool().AddFunc(compress)); - return promise.GetFuture(); -} - -void TCompressingProducer::Compress(NThreading::TPromise<TData>& promise, TData&& data, ECodec defaultCodec, int quality, const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib) { - if (!self.lock()) { - return; - } - promise.SetValue(TData::Encode(std::move(data), defaultCodec, quality)); - SignalProcessQueue(queueTag, self, pqLib); -} - -void TCompressingProducer::SignalProcessQueue(const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib) { - auto processQueue = [self] { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->ProcessQueue(); - } - }; - - //can't use this->Logger here - weak_ptr::lock -> ~TCompressingProducer in compression thread -> - //race with cancel() in main thread pool - Y_UNUSED(pqLib->GetQueuePool().GetQueue(queueTag).AddFunc(processQueue)); -} - -void TCompressingProducer::ProcessQueue() { - if (DestroyedFlag) { - return; - } - - size_t removeCount = 0; - bool prevSignalled = true; // flag to guarantee correct order of futures signalling. - for (TWriteRequestInfo& request : Queue) { - if (!request.EncodedData.HasValue()) { - // stage 1 - break; - } - if (!request.Future.Initialized()) { - // stage 2 - if (request.SeqNo) { - request.Future = Subproducer->Write(request.SeqNo, request.EncodedData.GetValue()); - } else { - request.Future = Subproducer->Write(request.EncodedData.GetValue()); - } - std::weak_ptr<TCompressingProducer> self = shared_from_this(); - const void* queueTag = this; - auto signalProcess = [self, queueTag, pqLib = PQLib.Get()](const auto&) { - SignalProcessQueue(queueTag, self, pqLib); - }; - request.Future.Subscribe(signalProcess); - } else { - // stage 3 - if (prevSignalled && request.Future.HasValue()) { - request.Promise.SetValue(request.Future.GetValue()); - ++removeCount; - } else { - prevSignalled = false; - } - } - } - while (removeCount--) { - Queue.pop_front(); - } -} - -void TCompressingProducer::DestroyQueue(const TString& reason) { - if (DestroyedFlag) { - return; - } - - DestroyedFlag = true; - for (TWriteRequestInfo& request : Queue) { - if (request.Future.Initialized() && request.Future.HasValue()) { - request.Promise.SetValue(request.Future.GetValue()); - } else { - TWriteResponse resp; - TError& err = *resp.MutableError(); - err.SetCode(NErrorCode::ERROR); - err.SetDescription(reason); - request.Promise.SetValue(TProducerCommitResponse(request.SeqNo, std::move(request.Data), std::move(resp))); - } - } - Queue.clear(); - - DestroyPQLibRef(); -} - -void TCompressingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept { - if (DestroyedFlag) { - return; - } - - Queue.emplace_back(seqNo, data, promise); - if (data.IsEncoded()) { - Queue.back().EncodedData = NThreading::MakeFuture<TData>(data); - ProcessQueue(); - } else if (DefaultCodec == ECodec::RAW) { - Queue.back().EncodedData = NThreading::MakeFuture<TData>(TData::MakeRawIfNotEncoded(data)); - ProcessQueue(); - } else { - Queue.back().EncodedData = Enqueue(std::move(data)); - } -} - -void TCompressingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept { - TCompressingProducer::Write(promise, 0, std::move(data)); -} - -NThreading::TFuture<TError> TCompressingProducer::IsDead() noexcept { - return Subproducer->IsDead(); -} - -NThreading::TFuture<void> TCompressingProducer::Destroyed() noexcept { - return WaitExceptionOrAll(DestroyedPromise.GetFuture(), Subproducer->Destroyed()); -} - -void TCompressingProducer::Cancel() { - DestroyQueue(GetCancelReason()); -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h deleted file mode 100644 index 6ed9e89bed0..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include "internals.h" -#include "iproducer_p.h" - -#include <deque> -#include <memory> - -namespace NPersQueue { - -class TPQLibPrivate; - -class TCompressingProducer: public IProducerImpl, public std::enable_shared_from_this<TCompressingProducer> { -public: - NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; - using IProducerImpl::Write; - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override; - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; - NThreading::TFuture<TError> IsDead() noexcept override; - NThreading::TFuture<void> Destroyed() noexcept override; - - ~TCompressingProducer(); - - TCompressingProducer(std::shared_ptr<IProducerImpl> subproducer, ECodec defaultCodec, int quality, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); - - void Cancel() override; - -private: - struct TWriteRequestInfo { - TWriteRequestInfo(TProducerSeqNo seqNo, TData data, const NThreading::TPromise<TProducerCommitResponse>& promise) - : SeqNo(seqNo) - , Data(std::move(data)) - , Promise(promise) - { - } - - TProducerSeqNo SeqNo; - TData Data; - NThreading::TFuture<TData> EncodedData; - NThreading::TPromise<TProducerCommitResponse> Promise; - NThreading::TFuture<TProducerCommitResponse> Future; - }; - - void ProcessQueue(); - NThreading::TFuture<TData> Enqueue(TData data); - // these functions should be static and not lock self in compression thread pool for proper futures ordering - static void Compress(NThreading::TPromise<TData>& promise, TData&& data, ECodec defaultCodec, int quality, - const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib); - static void SignalProcessQueue(const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib); - void DestroyQueue(const TString& reason); - -protected: - TIntrusivePtr<ILogger> Logger; - std::shared_ptr<IProducerImpl> Subproducer; - ECodec DefaultCodec; - int Quality; - std::deque<TWriteRequestInfo> Queue; - bool DestroyedFlag = false; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp deleted file mode 100644 index 4d9b30f69aa..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp +++ /dev/null @@ -1,243 +0,0 @@ -#include "compressing_producer.h" -#include "local_caller.h" -#include "persqueue_p.h" - -#include <library/cpp/testing/gmock_in_unittest/gmock.h> -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/thread.h> - -using namespace testing; - -namespace NPersQueue { - -class TMockProducer: public IProducerImpl { -public: - TMockProducer() - : IProducerImpl(nullptr, nullptr) - { - } - - NThreading::TFuture<TProducerCreateResponse> Start(TInstant) noexcept override { - return NThreading::MakeFuture<TProducerCreateResponse>(MockStart()); - } - - NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept override { - return NThreading::MakeFuture<TProducerCommitResponse>(MockWriteWithSeqNo(seqNo, data)); - } - - NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept override { - return NThreading::MakeFuture<TProducerCommitResponse>(MockWrite(data)); - } - - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override { - promise.SetValue(MockWriteWithSeqNo(seqNo, data)); - } - - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override { - promise.SetValue(MockWrite(data)); - } - - NThreading::TFuture<TError> IsDead() noexcept override { - return MockIsDead(); - } - - void Cancel() override { - } - - MOCK_METHOD(TProducerCreateResponse, MockStart, (), ()); - MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ()); - MOCK_METHOD(TProducerCommitResponse, MockWriteWithSeqNo, (TProducerSeqNo, TData), ()); - MOCK_METHOD(TProducerCommitResponse, MockWrite, (TData), ()); -}; - -template <class TMock = TMockProducer> -struct TCompressingProducerBootstrap { - ~TCompressingProducerBootstrap() { - Lib->CancelObjectsAndWait(); - } - - void Create(ECodec codec = ECodec::GZIP) { - Lib = new TPQLibPrivate(Settings); - MockProducer = std::make_shared<TMock>(); - CompressingProducer = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(MockProducer, codec, 3, Lib->GetSelfRefsAreDeadPtr(), Lib, Logger); - Lib->AddToDestroySet(CompressingProducer); - } - - void MakeOKStartResponse() { - TWriteResponse wresp; - wresp.MutableInit()->SetTopic("trololo"); - TProducerCreateResponse resp(std::move(wresp)); - EXPECT_CALL(*MockProducer, MockStart()) - .WillOnce(Return(resp)); - } - - void ExpectIsDeadCall() { - EXPECT_CALL(*MockProducer, MockIsDead()) - .WillOnce(Return(DeadPromise.GetFuture())); - } - - void Start() { - MakeOKStartResponse(); - UNIT_ASSERT_STRINGS_EQUAL(CompressingProducer->Start().GetValueSync().Response.GetInit().GetTopic(), "trololo"); - } - - TPQLibSettings Settings; - TIntrusivePtr<TCerrLogger> Logger = new TCerrLogger(TLOG_DEBUG); - TIntrusivePtr<TPQLibPrivate> Lib; - std::shared_ptr<TMock> MockProducer; - std::shared_ptr<IProducerImpl> CompressingProducer; - NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); -}; - -Y_UNIT_TEST_SUITE(TCompressingProducerTest) { - Y_UNIT_TEST(PassesStartAndDead) { - TCompressingProducerBootstrap<> bootstrap; - bootstrap.Create(); - bootstrap.ExpectIsDeadCall(); - - TWriteResponse wresp; - wresp.MutableError()->SetDescription("trololo"); - TProducerCreateResponse resp(std::move(wresp)); - EXPECT_CALL(*bootstrap.MockProducer, MockStart()) - .WillOnce(Return(resp)); - - auto startFuture = bootstrap.CompressingProducer->Start(); - UNIT_ASSERT_STRINGS_EQUAL(startFuture.GetValueSync().Response.GetError().GetDescription(), "trololo"); - - auto isDeadFuture = bootstrap.CompressingProducer->IsDead(); - UNIT_ASSERT(isDeadFuture.Initialized()); - UNIT_ASSERT(!isDeadFuture.HasValue()); - - TError err; - err.SetDescription("42"); - bootstrap.DeadPromise.SetValue(err); - - UNIT_ASSERT_STRINGS_EQUAL(isDeadFuture.GetValueSync().GetDescription(), "42"); - } - - Y_UNIT_TEST(CallsWritesInProperOrder) { - TCompressingProducerBootstrap<> bootstrap; - bootstrap.Create(); - bootstrap.Start(); - - TData data1 = TData("data1"); - TData expectedData1 = TData::Encode(data1, ECodec::GZIP, 3); - - TData data100 = TData::Raw("data100"); - TData expectedData100 = data100; - - TData data101 = TData("data101", ECodec::LZOP); - TData expectedData101 = TData::Encode(data101, ECodec::GZIP, 3); - - TWriteResponse wresp; - wresp.MutableAck(); - - InSequence seq; - EXPECT_CALL(*bootstrap.MockProducer, MockWriteWithSeqNo(1, expectedData1)) - .WillOnce(Return(TProducerCommitResponse(1, expectedData1, TWriteResponse(wresp)))); - - EXPECT_CALL(*bootstrap.MockProducer, MockWriteWithSeqNo(100, expectedData100)) - .WillOnce(Return(TProducerCommitResponse(100, expectedData100, TWriteResponse(wresp)))); - - EXPECT_CALL(*bootstrap.MockProducer, MockWrite(expectedData101)) - .WillOnce(Return(TProducerCommitResponse(101, expectedData101, TWriteResponse(wresp)))); - - auto write1 = bootstrap.CompressingProducer->Write(1, data1); - auto write100 = bootstrap.CompressingProducer->Write(100, data100); - auto write101 = bootstrap.CompressingProducer->Write(data101); - - write1.GetValueSync(); - write100.GetValueSync(); - write101.GetValueSync(); - } - - class TMockProducerWithReorder: public TMockProducer { - public: - using TMockProducer::TMockProducer; - using TMockProducer::Write; - - NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept override { - return PromiseWriteWithSeqNo(seqNo, data); - } - - MOCK_METHOD(NThreading::TFuture<TProducerCommitResponse>, PromiseWriteWithSeqNo, (TProducerSeqNo, TData), ()); - }; - - void SignalsAllFuturesInProperOrderImpl(bool death) { - size_t lastWritten = 0; - - TCompressingProducerBootstrap<TMockProducerWithReorder> bootstrap; - bootstrap.Create(); - bootstrap.Start(); - - const size_t count = 200; - NThreading::TFuture<TProducerCommitResponse> responses[count]; - TData datas[count]; - - TWriteResponse wresp; - wresp.MutableAck(); - EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(_, _)) - .Times(AtLeast(0)) - .WillRepeatedly(Return(NThreading::MakeFuture<TProducerCommitResponse>(TProducerCommitResponse(1, TData("data"), TWriteResponse(wresp))))); - - NThreading::TPromise<TProducerCommitResponse> promise = NThreading::NewPromise<TProducerCommitResponse>(); - - if (death) { - EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(50, _)) - .Times(AtLeast(0)) - .WillRepeatedly(Return(promise.GetFuture())); - } else { - EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(50, _)) - .Times(1) - .WillRepeatedly(Return(promise.GetFuture())); - } - - TString bigString(10000, 'A'); - for (size_t i = 0; i < count; ++i) { - TData data(TStringBuilder() << bigString << i, ECodec::LZOP); - datas[i] = data; - } - - auto testThreadId = TThread::CurrentThreadId(), prevThreadId = 0ul; - for (size_t i = 0; i < count; ++i) { - size_t seqNo = i + 1; - responses[i] = bootstrap.CompressingProducer->Write(i + 1, datas[i]); - auto onSignal = [&lastWritten, &prevThreadId, seqNo, testThreadId](const auto&) { - // proper thread - auto curId = TThread::CurrentThreadId(); - if (prevThreadId != 0) { - UNIT_ASSERT_C(curId == prevThreadId || curId == testThreadId, // could be executed in unittest thread if future ia already signalled or in object thread - "prevThreadId: " << Hex(prevThreadId) << ", curId: " << Hex(curId) << ", testThreadId: " << Hex(testThreadId)); - } else if (curId != testThreadId) { - prevThreadId = curId; - } - - // proper order - UNIT_ASSERT_VALUES_EQUAL(seqNo, lastWritten + 1); - lastWritten = seqNo; - }; - responses[i].Subscribe(onSignal); - } - - if (death) { - bootstrap.CompressingProducer = nullptr; - } - - responses[45].GetValueSync(); - promise.SetValue(TProducerCommitResponse(49, TData("data"), TWriteResponse(wresp))); - responses[count - 1].GetValueSync(); - for (size_t i = count - 1; i > 0; --i) { - UNIT_ASSERT(responses[i - 1].HasValue()); - } - } - - Y_UNIT_TEST(SignalsAllFuturesInProperOrderDuringDeath) { - SignalsAllFuturesInProperOrderImpl(true); - } - - Y_UNIT_TEST(SignalsAllFuturesInProperOrder) { - SignalsAllFuturesInProperOrderImpl(false); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp deleted file mode 100644 index da5536ab6fd..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp +++ /dev/null @@ -1,612 +0,0 @@ -#include "consumer.h" -#include "channel.h" -#include "persqueue_p.h" - -#include <kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -#include <util/generic/strbuf.h> -#include <util/string/cast.h> -#include <util/string/printf.h> -#include <util/string/vector.h> -#include <util/string/builder.h> - -#include <grpc++/create_channel.h> - -namespace NPersQueue { - -class TConsumerDestroyHandler : public IHandler { -public: - TConsumerDestroyHandler(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : Ptr(std::move(ptr)) - , RpcStuff(std::move(rpcStuff)) - , QueueTag(queueTag) - , PQLib(pqLib) - {} - - void Destroy(const TError& reason) override { - auto consumer = Ptr; - auto handler = [consumer, reason] { - auto consumerShared = consumer.lock(); - if (consumerShared) { - consumerShared->Destroy(reason); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - - TString ToString() override { return "DestroyHandler"; } - -protected: - std::weak_ptr<TConsumer> Ptr; - TIntrusivePtr<TConsumer::TRpcStuff> RpcStuff; // must simply live - const void* QueueTag; - TPQLibPrivate* PQLib; -}; - -class TConsumerStreamCreated : public TConsumerDestroyHandler { -public: - TConsumerStreamCreated(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) - {} - - void Done() override { - auto consumer = Ptr; - auto handler = [consumer] { - auto consumerShared = consumer.lock(); - if (consumerShared) { - consumerShared->OnStreamCreated(); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - - TString ToString() override { return "StreamCreated"; } -}; - -class TConsumerWriteDone : public TConsumerDestroyHandler { -public: - TConsumerWriteDone(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) - {} - - void Done() override { - auto consumer = Ptr; - auto handler = [consumer] { - auto consumerShared = consumer.lock(); - if (consumerShared) { - consumerShared->OnWriteDone(); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - - TString ToString() override { return "WriteDone"; } -}; - -class TConsumerReadDone : public TConsumerDestroyHandler { -public: - TConsumerReadDone(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) - {} - - void Done() override { - auto consumer = Ptr; - auto handler = [consumer] { - auto consumerShared = consumer.lock(); - if (consumerShared) { - consumerShared->OnReadDone(); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - - TString ToString() override { return "ReadDone"; } -}; - -void TConsumer::Destroy() noexcept { - IsDestroying = true; - RpcStuff->Context.TryCancel(); - ChannelHolder.ChannelPtr = nullptr; //if waiting for channel creation - Destroy("Destructor called"); -} - -TConsumer::TConsumer(const TConsumerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, - NThreading::TPromise<TConsumerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept - : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) - , RpcStuff(new TRpcStuff()) - , Settings(settings) - , StartPromise(promise) - , IsDeadPromise(NThreading::NewPromise<TError>()) - , UncommittedCount(0) - , UncommittedSize(0) - , MemoryUsage(0) - , ReadsOrdered(0) - , EstimateReadSize(settings.MaxSize) - , WriteInflight(true) - , ProxyCookie(0) - , Logger(std::move(logger)) - , IsDestroyed(false) - , IsDestroying(false) -{ - RpcStuff->CQ = std::move(cq); -} - -void TConsumer::Init() -{ - std::weak_ptr<TConsumer> self(shared_from_this()); - StreamCreatedHandler.Reset(new TConsumerStreamCreated(self, RpcStuff, this, PQLib.Get())); - ReadDoneHandler.Reset(new TConsumerReadDone(self, RpcStuff, this, PQLib.Get())); - WriteDoneHandler.Reset(new TConsumerWriteDone(self, RpcStuff, this, PQLib.Get())); -} - -void TConsumer::DoStart(TInstant deadline) -{ - if (IsDestroyed) - return; - - if (deadline != TInstant::Max()) { - std::weak_ptr<TConsumer> self = shared_from_this(); - auto onStartTimeout = [self] { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnStartTimeout(); - } - }; - StartDeadlineCallback = - PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout); - } - - FillMetaHeaders(RpcStuff->Context, Settings.Server.Database, Settings.CredentialsProvider.get()); - RpcStuff->Stub = PersQueueService::NewStub(RpcStuff->Channel); - RpcStuff->Stream = RpcStuff->Stub->AsyncReadSession(&RpcStuff->Context, RpcStuff->CQ.get(), new TQueueEvent(StreamCreatedHandler)); -} - -NThreading::TFuture<TConsumerCreateResponse> TConsumer::Start(TInstant deadline) noexcept -{ - NThreading::TFuture<TConsumerCreateResponse> ret = StartPromise.GetFuture(); - if (ChannelHolder.ChannelInfo.Initialized()) { - std::weak_ptr<TConsumer> self = shared_from_this(); - PQLib->Subscribe(ChannelHolder.ChannelInfo, - this, - [self, deadline](const auto& f) { - auto sharedSelf = self.lock(); - if (sharedSelf) { - sharedSelf->SetChannel(f.GetValue()); - sharedSelf->DoStart(deadline); - } - }); - } else { - if (RpcStuff->Channel && ProxyCookie) { - DoStart(deadline); - } else { - Destroy("No channel"); - } - } - return ret; -} - -void TConsumer::SetChannel(const TChannelHolder& channel) noexcept { - if (IsDestroyed) - return; - - ChannelHolder = channel; -} - -void TConsumer::SetChannel(const TChannelInfo& channel) noexcept { - if (!channel.Channel || channel.ProxyCookie == 0) { - Destroy("Channel creation error"); - return; - } - - if (IsDestroyed) - return; - Y_VERIFY(!RpcStuff->Channel); - RpcStuff->Channel = channel.Channel; - ProxyCookie = channel.ProxyCookie; -} - - -TConsumer::~TConsumer() noexcept { - Destroy(); -} - - -/************************************************read*****************************************************/ - -void TConsumer::OrderRead() noexcept { - if (IsDeadPromise.HasValue()) { - return; - } - - if (StartPromise.Initialized()) { - return; - } - - if (MemoryUsage >= Settings.MaxMemoryUsage || ReadsOrdered >= Settings.MaxInflyRequests) { - return; - } - if (Settings.MaxUncommittedCount > 0 && UncommittedCount >= Settings.MaxUncommittedCount) { - return; - } - if (Settings.MaxUncommittedSize > 0 && UncommittedSize >= Settings.MaxUncommittedSize) { - return; - } - - - if (IsDeadPromise.HasValue()) { - return; - } else { - while (ReadsOrdered < Settings.MaxInflyRequests && MemoryUsage + ((ui64)ReadsOrdered) * EstimateReadSize <= Settings.MaxMemoryUsage) { - TReadRequest request; - Settings.CredentialsProvider->FillAuthInfo(request.MutableCredentials()); - request.MutableRead()->SetMaxCount(Settings.MaxCount); - request.MutableRead()->SetMaxSize(Settings.MaxSize); - request.MutableRead()->SetPartitionsAtOnce(Settings.PartsAtOnce); - request.MutableRead()->SetMaxTimeLagMs(Settings.MaxTimeLagMs); - request.MutableRead()->SetReadTimestampMs(Settings.ReadTimestampMs); - Requests.emplace_back(std::move(request)); - ++ReadsOrdered; - DEBUG_LOG("read ordered memusage " << MemoryUsage << " infly " << ReadsOrdered << " maxmemusage " - << Settings.MaxMemoryUsage, "", SessionId); - } - - DoWrite(); - } -} - -void TConsumer::OnWriteDone() { - WriteInflight = false; - DoWrite(); -} - -void TConsumer::DoWrite() { - if (IsDestroyed) { - return; - } - - if (WriteInflight || Requests.empty()) - return; - - WriteInflight = true; - TReadRequest req = Requests.front(); - Requests.pop_front(); - DEBUG_LOG("sending request " << req.GetRequestCase(), "", SessionId); - RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); -} - - -void TConsumer::OnStreamCreated() { - if (IsDestroyed) { - return; - } - - TReadRequest req; - Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials()); - - TReadRequest::TInit* const init = req.MutableInit(); - init->SetClientId(Settings.ClientId); - for (const auto& t : Settings.Topics) { - init->AddTopics(t); - } - init->SetReadOnlyLocal(!Settings.ReadMirroredPartitions); - init->SetClientsideLocksAllowed(Settings.UseLockSession); - init->SetProxyCookie(ProxyCookie); - init->SetProtocolVersion(TReadRequest::ReadParamsInInit); - init->SetBalancePartitionRightNow(Settings.BalanceRightNow); - init->SetCommitsDisabled(Settings.CommitsDisabled); - - for (auto g : Settings.PartitionGroups) { - init->AddPartitionGroups(g); - } - - // Read settings - init->SetMaxReadMessagesCount(Settings.MaxCount); - init->SetMaxReadSize(Settings.MaxSize); - init->SetMaxReadPartitionsCount(Settings.PartsAtOnce); - init->SetMaxTimeLagMs(Settings.MaxTimeLagMs); - init->SetReadTimestampMs(Settings.ReadTimestampMs); - - WriteInflight = true; - RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); - - RpcStuff->Response.Clear(); - RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); -} - -void TConsumer::OnReadDone() { - if (RpcStuff->Response.HasError()) { - Destroy(RpcStuff->Response.GetError()); - return; - } - - if (IsDeadPromise.HasValue()) - return; - if (StartPromise.Initialized()) { //init response - NThreading::TPromise<TConsumerCreateResponse> tmp; - tmp.Swap(StartPromise); - - Y_VERIFY(RpcStuff->Response.HasInit()); - auto res(std::move(RpcStuff->Response)); - RpcStuff->Response.Clear(); - RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); - - SessionId = res.GetInit().GetSessionId(); - DEBUG_LOG("read stream created", "", SessionId); - - tmp.SetValue(TConsumerCreateResponse{std::move(res)}); - - OrderRead(); - - return; - } - if (RpcStuff->Response.HasBatchedData()) { - ConvertToOldBatch(RpcStuff->Response); // LOGBROKER-3173 - } - if (RpcStuff->Response.HasData()) { //read response - NThreading::TPromise<TConsumerMessage> p(NThreading::NewPromise<TConsumerMessage>()); - MessageResponses.push_back(p.GetFuture()); - const ui32 sz = RpcStuff->Response.ByteSize(); - MemoryUsage += sz; - Y_VERIFY(ReadsOrdered); - --ReadsOrdered; - - ui32 cnt = 0; - for (ui32 i = 0; i < RpcStuff->Response.GetData().MessageBatchSize(); ++i) { - cnt += RpcStuff->Response.GetData().GetMessageBatch(i).MessageSize(); - } - if (!Settings.CommitsDisabled) { - ReadInfo.push_back({RpcStuff->Response.GetData().GetCookie(), {cnt, sz}}); - UncommittedSize += sz; - UncommittedCount += cnt; - } - - DEBUG_LOG("read done memusage " << MemoryUsage << " infly " << ReadsOrdered << " size " << sz - << " ucs " << UncommittedSize << " ucc " << UncommittedCount, "", SessionId); - - auto res(std::move(RpcStuff->Response)); - - RpcStuff->Response.Clear(); - RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); - - OrderRead(); - - Y_VERIFY(MemoryUsage >= sz); - EstimateReadSize = res.ByteSize(); - OrderRead(); - - p.SetValue(TConsumerMessage{std::move(res)}); - } else if (RpcStuff->Response.HasLock() || RpcStuff->Response.HasRelease() || RpcStuff->Response.HasCommit() || RpcStuff->Response.HasPartitionStatus()) { //lock/release/commit response - if (!RpcStuff->Response.HasLock()) { - INFO_LOG("got from server " << RpcStuff->Response, "", SessionId); - if (RpcStuff->Response.HasCommit() && !Settings.CommitsDisabled) { - for (ui32 i = 0; i < RpcStuff->Response.GetCommit().CookieSize(); ++i) { - ui64 cookie = RpcStuff->Response.GetCommit().GetCookie(i); - Y_VERIFY(!ReadInfo.empty() && ReadInfo.front().first == cookie); - Y_VERIFY(UncommittedCount >= ReadInfo.front().second.first); - Y_VERIFY(UncommittedSize >= ReadInfo.front().second.second); - UncommittedCount -= ReadInfo.front().second.first; - UncommittedSize -= ReadInfo.front().second.second; - ReadInfo.pop_front(); - } - } - MessageResponses.push_back(NThreading::MakeFuture<TConsumerMessage>(TConsumerMessage(std::move(RpcStuff->Response)))); - OrderRead(); - } else { - const auto& p = RpcStuff->Response.GetLock(); - NThreading::TPromise<TLockInfo> promise(NThreading::NewPromise<TLockInfo>()); - //TODO: add subscribe - auto f = promise.GetFuture(); - std::weak_ptr<TConsumer> ptr = shared_from_this(); - - //will take ptr for as long as promises are not set - PQLib->Subscribe(f, - this, - [ptr, p](const auto& f) { - auto consumer = ptr.lock(); - if (consumer) { - consumer->Lock(p.GetTopic(), p.GetPartition(), p.GetGeneration(), f.GetValue().ReadOffset, f.GetValue().CommitOffset, - f.GetValue().VerifyReadOffset); - } - }); - - INFO_LOG("got LOCK from server " << p, "", SessionId); - MessageResponses.push_back(NThreading::MakeFuture<TConsumerMessage>(TConsumerMessage(std::move(RpcStuff->Response), std::move(promise)))); - } - - RpcStuff->Response.Clear(); - RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); - } else { - Y_FAIL("unsupported response %s", RpcStuff->Response.DebugString().c_str()); - } - - ProcessResponses(); -} - - -/******************************************commit****************************************************/ -void TConsumer::Commit(const TVector<ui64>& cookies) noexcept { - auto sortedCookies = cookies; - std::sort(sortedCookies.begin(), sortedCookies.end()); - - TReadRequest request; - for (const auto& cookie : sortedCookies) { - request.MutableCommit()->AddCookie(cookie); - } - - DEBUG_LOG("sending COMMIT to server " << request.GetCommit(), "", SessionId); - if (IsDestroyed || StartPromise.Initialized()) { - ERR_LOG("trying to commit " << (IsDestroyed ? "after destroy" : "before start"), "", SessionId); - return; - } - Requests.emplace_back(std::move(request)); - DoWrite(); -} - - -void TConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { - TReadRequest request; - auto status = request.MutableStatus(); - status->SetTopic(topic); - status->SetPartition(partition); - status->SetGeneration(generation); - - DEBUG_LOG("sending GET_STATUS to server " << request.GetStatus(), "", SessionId); - if (IsDestroyed || StartPromise.Initialized()) { - ERR_LOG("trying to get status " << (IsDestroyed ? "after destroy" : "before start"), "", SessionId); - return; - } - Requests.emplace_back(std::move(request)); - DoWrite(); -} - -void TConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { - if (IsDeadPromise.HasValue()) { - TReadResponse res; - res.MutableError()->SetDescription("consumer is dead"); - res.MutableError()->SetCode(NErrorCode::ERROR); - promise.SetValue(TConsumerMessage{std::move(res)}); - return; - } - - if (StartPromise.Initialized()) { - TReadResponse res; - res.MutableError()->SetDescription("consumer is not ready"); - res.MutableError()->SetCode(NErrorCode::ERROR); - promise.SetValue(TConsumerMessage{std::move(res)}); - return; - } - - MessagePromises.push_back(promise); - ProcessResponses(); //if alredy have an answer -} - - -void TConsumer::Lock(const TString& topic, const ui32 partition, const ui64 generation, const ui64 readOffset, - const ui64 commitOffset, const bool verifyReadOffset) noexcept { - Y_VERIFY(Settings.UseLockSession); - - TReadRequest request; - request.MutableStartRead()->SetTopic(topic); - request.MutableStartRead()->SetPartition(partition); - request.MutableStartRead()->SetReadOffset(readOffset); - request.MutableStartRead()->SetCommitOffset(commitOffset); - request.MutableStartRead()->SetVerifyReadOffset(verifyReadOffset); - request.MutableStartRead()->SetGeneration(generation); - INFO_LOG("sending START_READ to server " << request.GetStartRead(), "", SessionId); - - Settings.CredentialsProvider->FillAuthInfo(request.MutableCredentials()); - - if (IsDestroyed) - return; - Requests.emplace_back(std::move(request)); - DoWrite(); -} - - -void TConsumer::ProcessResponses() { - while(true) { - if (MessagePromises.empty() || MessageResponses.empty() || !MessageResponses.front().HasValue()) - break; - auto p(std::move(MessagePromises.front())); - MessagePromises.pop_front(); - auto res(MessageResponses.front().ExtractValue()); - MessageResponses.pop_front(); - if (res.Type == EMT_DATA) { - ui32 sz = res.Response.ByteSize(); - Y_VERIFY(MemoryUsage >= sz); - MemoryUsage -= sz; - } - - if (res.Type == EMT_DATA) { - OrderRead(); - } - - p.SetValue(TConsumerMessage{std::move(res)}); - } -} - -/***********************************************************************************/ - -NThreading::TFuture<TError> TConsumer::IsDead() noexcept { - return IsDeadPromise.GetFuture(); -} - -void TConsumer::Destroy(const TString& description) { - TError error; - error.SetDescription(description); - error.SetCode(NErrorCode::ERROR); - Destroy(error); -} - -void TConsumer::Destroy(const TError& error) { - if (!IsDestroying) { - INFO_LOG("error: " << error, "", SessionId); - } - - if (IsDestroyed) - return; - IsDestroyed = true; - - IsDeadPromise.SetValue(error); - - MessageResponses.clear(); - - NThreading::TFuture<TChannelInfo> tmp; - tmp.Swap(ChannelHolder.ChannelInfo); - ChannelHolder.ChannelPtr = nullptr; - - Error = error; - if (StartPromise.Initialized()) { - NThreading::TPromise<TConsumerCreateResponse> tmp; - tmp.Swap(StartPromise); - TReadResponse res; - res.MutableError()->CopyFrom(Error); - tmp.SetValue(TConsumerCreateResponse{std::move(res)}); - } - - while (!MessagePromises.empty()) { - auto p = MessagePromises.front(); - MessagePromises.pop_front(); - TReadResponse res; - res.MutableError()->CopyFrom(Error); - p.SetValue(TConsumerMessage{std::move(res)}); - } - - StreamCreatedHandler.Reset(); - ReadDoneHandler.Reset(); - WriteDoneHandler.Reset(); - - DestroyPQLibRef(); -} - -void TConsumer::OnStartTimeout() { - if (IsDestroyed) { - return; - } - - StartDeadlineCallback = nullptr; - if (!StartPromise.Initialized()) { - // already replied with successful start - return; - } - - TError error; - error.SetDescription("Start timeout"); - error.SetCode(NErrorCode::CREATE_TIMEOUT); - Destroy(error); -} - -void TConsumer::Cancel() { - IsDestroying = true; - RpcStuff->Context.TryCancel(); - ChannelHolder.ChannelPtr = nullptr; //if waiting for channel creation - - Destroy(GetCancelReason()); -} - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h deleted file mode 100644 index f2f6f81e714..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h +++ /dev/null @@ -1,121 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include "channel.h" -#include "scheduler.h" -#include "iconsumer_p.h" -#include "internals.h" -#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> - -#include <library/cpp/threading/future/future.h> - -#include <deque> -#include <memory> - -namespace NPersQueue { - -class TConsumer: public IConsumerImpl, public std::enable_shared_from_this<TConsumer> { -public: - friend class TPQLibPrivate; - - NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override; - - void Commit(const TVector<ui64>& cookies) noexcept override; - - using IConsumerImpl::GetNextMessage; - void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; - - void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; - - NThreading::TFuture<TError> IsDead() noexcept override; - - TConsumer(const TConsumerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, - NThreading::TPromise<TConsumerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept; - - void Init() override; - - ~TConsumer() noexcept; - - void Destroy() noexcept; - void SetChannel(const TChannelHolder& channel) noexcept; - void SetChannel(const TChannelInfo& channel) noexcept; - - void Lock(const TString& topic, const ui32 partition, const ui64 generation, const ui64 readOffset, const ui64 commitOffset, const bool verifyReadOffset) noexcept; - void OrderRead() noexcept; - - void Cancel() override; - -public: - using TStream = grpc::ClientAsyncReaderWriterInterface<TReadRequest, TReadResponse>; - - // objects that must live after destruction of producer untill all the callbacks arrive at CompletionQueue - struct TRpcStuff: public TAtomicRefCount<TRpcStuff> { - TReadResponse Response; - std::shared_ptr<grpc::CompletionQueue> CQ; - std::shared_ptr<grpc::Channel> Channel; - std::unique_ptr<PersQueueService::Stub> Stub; - grpc::ClientContext Context; - std::unique_ptr<TStream> Stream; - }; - -protected: - friend class TConsumerStreamCreated; - friend class TConsumerReadDone; - friend class TConsumerWriteDone; - friend class TConsumerDestroyHandler; - - IHandlerPtr StreamCreatedHandler; - IHandlerPtr ReadDoneHandler; - IHandlerPtr WriteDoneHandler; - - void ProcessResponses(); - void Destroy(const TError& reason); - void Destroy(const TString& description); // the same but with Code=ERROR - void OnStreamCreated(); - void OnReadDone(); - void OnWriteDone(); - - void DoWrite(); - void DoStart(TInstant deadline); - void OnStartTimeout(); - - TIntrusivePtr<TRpcStuff> RpcStuff; - - TChannelHolder ChannelHolder; - TConsumerSettings Settings; - - TString SessionId; - - NThreading::TPromise<TConsumerCreateResponse> StartPromise; - NThreading::TPromise<TError> IsDeadPromise; - std::deque<NThreading::TPromise<TConsumerMessage>> MessagePromises; - std::deque<NThreading::TFuture<TConsumerMessage>> MessageResponses; - - std::deque<TReadRequest> Requests; - - - std::deque<std::pair<ui64, std::pair<ui32, ui64>>> ReadInfo; //cookie -> (count, size) - ui32 UncommittedCount = 0; - ui64 UncommittedSize = 0; - - ui64 MemoryUsage = 0; - ui32 ReadsOrdered = 0; - - ui64 EstimateReadSize = 0; - - bool WriteInflight; - - ui64 ProxyCookie = 0; - - TIntrusivePtr<ILogger> Logger; - - TError Error; - - bool IsDestroyed; - bool IsDestroying; - - TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp deleted file mode 100644 index dc09bd2ef1c..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> - -#include <library/cpp/testing/unittest/registar.h> - -namespace NPersQueue { -Y_UNIT_TEST_SUITE(TConsumerTest) { - Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) { - // Test that consumer doesn't hang on till shutdown - TPQLib lib; - TConsumerSettings settings; - settings.Server = TServerSetting{"localhost"}; - settings.Topics.push_back("topic"); - settings.ClientId = "client"; - lib.CreateConsumer(settings, {}, false); - } - - Y_UNIT_TEST(StartTimeout) { - if (GrpcV1EnabledByDefault()) { - return; - } - - TTestServer testServer; - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic", 1); - testServer.WaitInit("topic"); - // Test that consumer doesn't hang on till shutdown - TConsumerSettings settings; - settings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - settings.Topics.push_back("topic"); - settings.ClientId = "client"; - auto consumer = testServer.PQLib->CreateConsumer(settings, {}, false); - auto startResult = consumer->Start(TInstant::Now()); - UNIT_ASSERT_EQUAL_C(startResult.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, startResult.GetValueSync().Response); - - DestroyAndWait(consumer); - } - - TProducerSettings MakeProducerSettings(const TTestServer& testServer) { - TProducerSettings producerSettings; - producerSettings.ReconnectOnFailure = false; - producerSettings.Topic = "topic1"; - producerSettings.SourceId = "123"; - producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - producerSettings.Codec = ECodec::LZOP; - return producerSettings; - } - - TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { - TConsumerSettings settings; - settings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - settings.Topics.emplace_back("topic1"); - settings.ClientId = "user"; - return settings; - } - - Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - - const size_t partitions = 1; - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - testServer.WaitInit("topic1"); - - - auto consumer = testServer.PQLib->CreateConsumer(MakeConsumerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false); - UNIT_ASSERT(!consumer->Start().GetValueSync().Response.HasError()); - auto isDead = consumer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - auto msg1 = consumer->GetNextMessage(); - auto msg2 = consumer->GetNextMessage(); - - testServer.PQLib = nullptr; - Cerr << "PQLib destroyed" << Endl; - - UNIT_ASSERT(msg1.HasValue()); - UNIT_ASSERT(msg2.HasValue()); - - UNIT_ASSERT(msg1.GetValue().Response.HasError()); - UNIT_ASSERT(msg2.GetValue().Response.HasError()); - - auto msg3 = consumer->GetNextMessage(); - UNIT_ASSERT(msg3.HasValue()); - UNIT_ASSERT(msg3.GetValue().Response.HasError()); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp deleted file mode 100644 index a9c99680811..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp +++ /dev/null @@ -1,349 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include "internals.h" - -#include <library/cpp/json/json_reader.h> -#include <library/cpp/http/simple/http_client.h> -#include <library/cpp/logger/priority.h> -#include <library/cpp/tvmauth/client/facade.h> - -#include <util/generic/ptr.h> -#include <util/string/builder.h> -#include <util/system/env.h> -#include <util/system/mutex.h> -#include <library/cpp/string_utils/quote/quote.h> -#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> -#include <kikimr/public/sdk/cpp/client/iam/iam.h> - -namespace NPersQueue { - -TString GetToken(ICredentialsProvider* credentials) { - NPersQueue::TCredentials auth; - credentials->FillAuthInfo(&auth); - TString token; - switch (auth.GetCredentialsCase()) { - case NPersQueue::TCredentials::kTvmServiceTicket: - token = auth.GetTvmServiceTicket(); - break; - case NPersQueue::TCredentials::kOauthToken: - token = auth.GetOauthToken(); - break; - case NPersQueue::TCredentials::CREDENTIALS_NOT_SET: - break; - default: - Y_VERIFY(true, "Unknown Credentials case."); - } - return token; -} - -const TString& GetTvmPqServiceName() { - static const TString PQ_SERVICE_NAME = "pq"; - return PQ_SERVICE_NAME; -} - -class TInsecureCredentialsProvider : public ICredentialsProvider { - void FillAuthInfo(NPersQueue::TCredentials*) const override {} -}; - -class TLogBridge: public NTvmAuth::ILogger { -public: - TLogBridge(TIntrusivePtr<NPersQueue::ILogger> logger = nullptr) - : Logger(logger) - {} - - void Log(int lvl, const TString& msg) override { - if (Logger) { - Logger->Log(msg, "", "", lvl); - } - } - -private: - TIntrusivePtr<NPersQueue::ILogger> Logger; -}; - -class TTVMCredentialsProvider : public ICredentialsProvider { -public: - TTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr) - : Alias(alias) - { - if (!settings.FetchServiceTicketsForDstsWithAliases.contains(Alias)) { - ythrow yexception() << "alias for `" << Alias << "` must be set"; - } - Logger = MakeIntrusive<TLogBridge>(std::move(logger)); - TvmClient = std::make_shared<NTvmAuth::TTvmClient>(settings, Logger); - } - - TTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr) - : TTVMCredentialsProvider(CreateSettings(secret, srcClientId, dstClientId, alias), alias, logger) - {} - - TTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr) - : TvmClient(tvmClient) - , Alias(alias) - , Logger(MakeIntrusive<TLogBridge>(std::move(logger))) - {} - - void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { - try { - auto status = TvmClient->GetStatus(); - if (status == NTvmAuth::TClientStatus::Ok) { - authInfo->SetTvmServiceTicket(TvmClient->GetServiceTicketFor(Alias)); - } else { - Logger->Error(TStringBuilder() << "Can't get ticket: " << status.GetLastError() << "\n"); - } - } catch (...) { - Logger->Error(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n"); - } - } - -private: - std::shared_ptr<NTvmAuth::TTvmClient> TvmClient; - TString Alias; - NTvmAuth::TLoggerPtr Logger; - - static NTvmAuth::NTvmApi::TClientSettings CreateSettings(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, const TString& alias) { - NTvmAuth::NTvmApi::TClientSettings settings; - settings.SetSelfTvmId(srcClientId); - settings.EnableServiceTicketsFetchOptions(secret, {{alias, dstClientId}}); - return settings; - } -}; - -class TTVMQloudCredentialsProvider : public ICredentialsProvider { -public: - TTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, const TDuration& refreshPeriod, TIntrusivePtr<ILogger> logger, ui32 port) - : HttpClient(TSimpleHttpClient("localhost", port)) - , Request(TStringBuilder() << "/tvm/tickets?src=" << CGIEscapeRet(srcAlias) << "&dsts=" << CGIEscapeRet(dstAlias)) - , DstAlias(dstAlias) - , LastTicketUpdate(TInstant::Zero()) - , RefreshPeriod(refreshPeriod) - , Logger(std::move(logger)) - , DstId(0) - { - GetTicket(); - } - - TTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, const TDuration& refreshPeriod, TIntrusivePtr<ILogger> logger, ui32 port) - : HttpClient(TSimpleHttpClient("localhost", port)) - , Request(TStringBuilder() << "/tvm/tickets?src=" << srcId << "&dsts=" << dstId) - , LastTicketUpdate(TInstant::Zero()) - , RefreshPeriod(refreshPeriod) - , Logger(std::move(logger)) - , DstId(dstId) - { - GetTicket(); - } - - void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { - if (TInstant::Now() > LastTicketUpdate + RefreshPeriod) { - GetTicket(); - } - - authInfo->SetTvmServiceTicket(Ticket); - } - -private: - TSimpleHttpClient HttpClient; - TString Request; - TString DstAlias; - mutable TString Ticket; - mutable TInstant LastTicketUpdate; - TDuration RefreshPeriod; - TIntrusivePtr<ILogger> Logger; - ui32 DstId; - - void GetTicket() const { - try { - TStringStream out; - TSimpleHttpClient::THeaders headers; - headers["Authorization"] = GetEnv("QLOUD_TVM_TOKEN"); - HttpClient.DoGet(Request, &out, headers); - NJson::TJsonValue resp; - NJson::ReadJsonTree(&out, &resp, true); - TString localDstAlias = DstAlias; - Y_VERIFY(!DstAlias.empty() || DstId != 0); - Y_VERIFY(resp.GetMap().size() == 1); - if (!localDstAlias.empty()) { - if (!resp.Has(localDstAlias)) { - ythrow yexception() << "Result doesn't contain dstAlias `" << localDstAlias << "`"; - } - } - else { - for (const auto &[k, v] : resp.GetMap()) { - if (!v.Has("tvm_id") || v["tvm_id"].GetIntegerSafe() != DstId) - ythrow yexception() << "Result doesn't contain dstId `" << DstId << "`"; - localDstAlias = k; - } - } - TString ticket = resp[localDstAlias]["ticket"].GetStringSafe(); - if (ticket.empty()) { - ythrow yexception() << "Got empty ticket"; - } - Ticket = ticket; - LastTicketUpdate = TInstant::Now(); - } catch (...) { - if (Logger) { - Logger->Log(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n", "", "", TLOG_ERR); - } - } - } -}; - -class TOAuthCredentialsProvider : public ICredentialsProvider { -public: - TOAuthCredentialsProvider(const TString& token) - : Token(token) - {} - - void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { - authInfo->SetOauthToken(Token); - } - -private: - TString Token; -}; - -class TTVMCredentialsForwarder : public ITVMCredentialsForwarder { -public: - TTVMCredentialsForwarder() = default; - - void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { - TGuard<TSpinLock> guard(Lock); - if (!Ticket.empty()) - authInfo->SetTvmServiceTicket(Ticket); - } - - void SetTVMServiceTicket(const TString& ticket) override { - TGuard<TSpinLock> guard(Lock); - Ticket = ticket; - } - - -private: - TString Ticket; - TSpinLock Lock; -}; - -class IIAMCredentialsProviderWrapper : public ICredentialsProvider { -public: - IIAMCredentialsProviderWrapper(std::shared_ptr<NYdb::ICredentialsProviderFactory> factory, TIntrusivePtr<ILogger> logger = nullptr) - : Provider(factory->CreateProvider()) - , Logger(std::move(logger)) - , Lock() - { - } - - void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { - TString ticket; - try { - with_lock(Lock) { - ticket = Provider->GetAuthInfo(); - } - } catch (...) { - if (Logger) { - Logger->Log(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n", "", "", TLOG_ERR); - } - return; - } - - authInfo->SetTvmServiceTicket(ticket); - } - -private: - std::shared_ptr<NYdb::ICredentialsProvider> Provider; - TIntrusivePtr<ILogger> Logger; - TMutex Lock; -}; - -class TIAMCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper { -public: - TIAMCredentialsProviderWrapper(TIntrusivePtr<ILogger> logger = nullptr) - : IIAMCredentialsProviderWrapper( - NYdb::CreateIamCredentialsProviderFactory(), - std::move(logger) - ) - { - } -}; - -class TIAMJwtFileCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper { -public: - TIAMJwtFileCredentialsProviderWrapper( - const TString& jwtKeyFilename, const TString& endpoint, const TDuration& refreshPeriod, - const TDuration& requestTimeout, TIntrusivePtr<ILogger> logger = nullptr - ) - : IIAMCredentialsProviderWrapper( - NYdb::CreateIamJwtFileCredentialsProviderFactory( - {{endpoint, refreshPeriod, requestTimeout}, jwtKeyFilename}), - std::move(logger) - ) - {} -}; - -class TIAMJwtParamsCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper { -public: - TIAMJwtParamsCredentialsProviderWrapper( - const TString& jwtParams, const TString& endpoint, const TDuration& refreshPeriod, - const TDuration& requestTimeout, TIntrusivePtr<ILogger> logger = nullptr - ) - : IIAMCredentialsProviderWrapper( - NYdb::CreateIamJwtParamsCredentialsProviderFactory( - {{endpoint, refreshPeriod, requestTimeout}, jwtParams}), - std::move(logger) - ) - {} -}; - -std::shared_ptr<ICredentialsProvider> CreateInsecureCredentialsProvider() { - return std::make_shared<TInsecureCredentialsProvider>(); -} - -std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, TIntrusivePtr<ILogger> logger) { - return std::make_shared<TTVMCredentialsProvider>(secret, srcClientId, dstClientId, GetTvmPqServiceName(), std::move(logger)); -} - -std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, TIntrusivePtr<ILogger> logger, const TString& alias) { - return std::make_shared<TTVMCredentialsProvider>(settings, alias, std::move(logger)); -} - -std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, TIntrusivePtr<ILogger> logger, const TString& alias) { - return std::make_shared<TTVMCredentialsProvider>(tvmClient, alias, std::move(logger)); -} - -std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, TIntrusivePtr<ILogger> logger, const TDuration refreshPeriod, ui32 port) { - return std::make_shared<TTVMQloudCredentialsProvider>(srcAlias, dstAlias, refreshPeriod, std::move(logger), port); -} - -std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, TIntrusivePtr<ILogger> logger, const TDuration refreshPeriod, ui32 port) { - return std::make_shared<TTVMQloudCredentialsProvider>(srcId, dstId, refreshPeriod, std::move(logger), port); -} - -std::shared_ptr<ICredentialsProvider> CreateOAuthCredentialsProvider(const TString& token) { - return std::make_shared<TOAuthCredentialsProvider>(token); -} - -std::shared_ptr<ITVMCredentialsForwarder> CreateTVMCredentialsForwarder() { - return std::make_shared<TTVMCredentialsForwarder>(); -} - -std::shared_ptr<ICredentialsProvider> CreateIAMCredentialsForwarder(TIntrusivePtr<ILogger> logger) { - return std::make_shared<TIAMCredentialsProviderWrapper>(logger); -} - -std::shared_ptr<ICredentialsProvider> CreateIAMJwtFileCredentialsForwarder( - const TString& jwtKeyFilename, TIntrusivePtr<ILogger> logger, - const TString& endpoint, const TDuration& refreshPeriod, const TDuration& requestTimeout -) { - return std::make_shared<TIAMJwtFileCredentialsProviderWrapper>(jwtKeyFilename, endpoint, - refreshPeriod, requestTimeout, logger); -} - -std::shared_ptr<ICredentialsProvider> CreateIAMJwtParamsCredentialsForwarder( - const TString& jwtParams, TIntrusivePtr<ILogger> logger, - const TString& endpoint, const TDuration& refreshPeriod, const TDuration& requestTimeout -) { - return std::make_shared<TIAMJwtParamsCredentialsProviderWrapper>(jwtParams, endpoint, - refreshPeriod, requestTimeout, logger); -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp deleted file mode 100644 index 0fbf62cf020..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp +++ /dev/null @@ -1,290 +0,0 @@ -#include "decompressing_consumer.h" -#include "persqueue_p.h" - -#include <library/cpp/streams/lzop/lzop.h> -#include <library/cpp/streams/zstd/zstd.h> - -#include <util/stream/mem.h> -#include <util/stream/zlib.h> - -#include <atomic> - -namespace NPersQueue { - -TDecompressingConsumer::TDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) - : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) - , Logger(std::move(logger)) - , Subconsumer(std::move(subconsumer)) - , Settings(settings) -{ -} - -void TDecompressingConsumer::Init() { - DestroyedFuture = WaitExceptionOrAll(DestroyedPromise.GetFuture(), Subconsumer->Destroyed()); -} - -TDecompressingConsumer::~TDecompressingConsumer() { - DestroyQueue("Destructor called"); -} - -template <class T> -void TDecompressingConsumer::SubscribeForQueueProcessing(NThreading::TFuture<T>& future) { - const void* queueTag = this; - std::weak_ptr<TDecompressingConsumer> self = shared_from_this(); - auto signalProcessQueue = [queueTag, self, pqLib = PQLib.Get()](const auto&) mutable { - SignalProcessQueue(queueTag, std::move(self), pqLib); - }; - future.Subscribe(signalProcessQueue); -} - -NThreading::TFuture<TConsumerCreateResponse> TDecompressingConsumer::Start(TInstant deadline) noexcept { - auto ret = Subconsumer->Start(deadline); - SubscribeForQueueProcessing(ret); // when subconsumer starts, we will read ahead some messages - - // subscribe to death - std::weak_ptr<TDecompressingConsumer> self = shared_from_this(); - auto isDeadHandler = [self](const auto& error) { - auto selfShared = self.lock(); - if (selfShared && !selfShared->IsDestroyed) { - selfShared->DestroyQueue(error.GetValue()); - } - }; - PQLib->Subscribe(Subconsumer->IsDead(), this, isDeadHandler); - - return ret; -} - -NThreading::TFuture<TError> TDecompressingConsumer::IsDead() noexcept { - return IsDeadPromise.GetFuture(); -} - -void TDecompressingConsumer::Commit(const TVector<ui64>& cookies) noexcept { - if (IsDestroyed) { - ERR_LOG("Attempt to commit to dead consumer.", "", ""); - return; - } - return Subconsumer->Commit(cookies); -} - -void TDecompressingConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { - if (IsDestroyed) { - ERR_LOG("Attempt to request status from dead consumer.", "", ""); - return; - } - return Subconsumer->RequestPartitionStatus(topic, partition, generation); -} - - -NThreading::TFuture<void> TDecompressingConsumer::Destroyed() noexcept { - return DestroyedFuture; -} - -void TDecompressingConsumer::AddNewGetNextMessageRequest(NThreading::TPromise<TConsumerMessage>& promise) { - Queue.emplace_back(promise); - Queue.back().Future = Subconsumer->GetNextMessage(); - SubscribeForQueueProcessing(Queue.back().Future); -} - -void TDecompressingConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { - if (IsDestroyed) { - TReadResponse response; - auto* err = response.MutableError(); - err->SetCode(NErrorCode::ERROR); - err->SetDescription("Consumer is dead."); - promise.SetValue(TConsumerMessage(std::move(response))); - return; - } - AddNewGetNextMessageRequest(promise); -} - -void TDecompressingConsumer::ProcessQueue() { - try { - // stage 1: wait subconsumer answer and run decompression tasks - for (TReadRequestInfo& request : Queue) { - if (!request.Data && request.Future.HasValue()) { - request.Data.ConstructInPlace(request.Future.ExtractValue()); - if (request.Data->Type == EMT_DATA) { - RequestDecompressing(request); - } - } - } - - // stage 2: answer ready tasks to client - while (!Queue.empty()) { - TReadRequestInfo& front = Queue.front(); - if (!front.Data || front.Data->Type == EMT_DATA && !front.AllDecompressing.HasValue()) { - break; - } - if (front.Data->Type == EMT_DATA) { - CopyDataToAnswer(front); - } - - front.Promise.SetValue(TConsumerMessage(std::move(*front.Data))); - Queue.pop_front(); - } - } catch (const std::exception&) { - DestroyQueue(TStringBuilder() << "Failed to decompress data: " << CurrentExceptionMessage()); - } -} - -void TDecompressingConsumer::SignalProcessQueue(const void* queueTag, std::weak_ptr<TDecompressingConsumer> self, TPQLibPrivate* pqLib) { - auto processQueue = [self] { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->ProcessQueue(); - } - }; - Y_VERIFY(pqLib->GetQueuePool().GetQueue(queueTag).AddFunc(processQueue)); -} - -namespace { - -struct TWaitAll: public TAtomicRefCount<TWaitAll> { - TWaitAll(size_t count) - : Count(count) - { - } - - void OnResultReady() { - if (--Count == 0) { - Promise.SetValue(); - } - } - - std::atomic<size_t> Count; - NThreading::TPromise<void> Promise = NThreading::NewPromise<void>(); -}; - -THolder<IInputStream> CreateDecompressor(const ECodec codec, IInputStream* origin) { - THolder<IInputStream> result; - if (codec == ECodec::GZIP) { - result.Reset(new TZLibDecompress(origin)); - } else if (codec == ECodec::LZOP) { - result.Reset(new TLzopDecompress(origin)); - } else if (codec == ECodec::ZSTD) { - result.Reset(new TZstdDecompress(origin)); - } - return result; -} - -} // namespace - -void TDecompressingConsumer::RequestDecompressing(TReadRequestInfo& request) { - size_t futuresCount = 0; - const TReadResponse::TData& data = request.Data->Response.GetData(); - request.BatchFutures.resize(data.MessageBatchSize()); - for (size_t i = 0; i < data.MessageBatchSize(); ++i) { - const TReadResponse::TData::TMessageBatch& batch = data.GetMessageBatch(i); - request.BatchFutures.reserve(batch.MessageSize()); - for (const TReadResponse::TData::TMessage& message : batch.GetMessage()) { - if (message.GetMeta().GetCodec() != ECodec::RAW) { - ++futuresCount; - request.BatchFutures[i].push_back(RequestDecompressing(message)); - } - } - } - - if (futuresCount == 0) { - request.AllDecompressing = NThreading::MakeFuture(); // done - return; - } else { - TIntrusivePtr<TWaitAll> waiter = new TWaitAll(futuresCount); - auto handler = [waiter](const auto&) { - waiter->OnResultReady(); - }; - for (auto& batch : request.BatchFutures) { - for (auto& future : batch) { - future.Subscribe(handler); - } - } - request.AllDecompressing = waiter->Promise.GetFuture(); - } - SubscribeForQueueProcessing(request.AllDecompressing); -} - -NThreading::TFuture<TString> TDecompressingConsumer::RequestDecompressing(const TReadResponse::TData::TMessage& message) { - TString data = message.GetData(); - ECodec codec = message.GetMeta().GetCodec(); - NThreading::TPromise<TString> promise = NThreading::NewPromise<TString>(); - auto decompress = [data, codec, promise]() mutable { - Decompress(data, codec, promise); - }; - Y_VERIFY(PQLib->GetCompressionPool().AddFunc(decompress)); - return promise.GetFuture(); -} - -void TDecompressingConsumer::Decompress(const TString& data, ECodec codec, NThreading::TPromise<TString>& promise) { - try { - // TODO: Check if decompression was successfull, i.e. if 'data' is valid byte array compressed with 'codec' - TMemoryInput iss(data.data(), data.size()); - THolder<IInputStream> dec = CreateDecompressor(codec, &iss); - if (!dec) { - ythrow yexception() << "Failed to create decompressor"; - } - - TString result; - TStringOutput oss(result); - TransferData(dec.Get(), &oss); - - promise.SetValue(result); - } catch (...) { - promise.SetException(std::current_exception()); - } -} - -void TDecompressingConsumer::CopyDataToAnswer(TReadRequestInfo& request) { - TReadResponse::TData& data = *request.Data->Response.MutableData(); - Y_VERIFY(request.BatchFutures.size() == data.MessageBatchSize()); - for (size_t i = 0; i < request.BatchFutures.size(); ++i) { - TReadResponse::TData::TMessageBatch& batch = *data.MutableMessageBatch(i); - auto currentMessage = request.BatchFutures[i].begin(); - for (TReadResponse::TData::TMessage& message : *batch.MutableMessage()) { - if (message.GetMeta().GetCodec() != ECodec::RAW) { - Y_VERIFY(currentMessage != request.BatchFutures[i].end()); - try { - message.SetData(currentMessage->GetValue()); - message.MutableMeta()->SetCodec(ECodec::RAW); - } catch (const std::exception& ex) { - if (Settings.SkipBrokenChunks) { - message.SetBrokenPackedData(message.GetData()); - message.SetData(""); - } else { - throw; - } - } - ++currentMessage; - } - } - Y_VERIFY(currentMessage == request.BatchFutures[i].end()); - } -} - -void TDecompressingConsumer::DestroyQueue(const TString& errorMessage) { - TError error; - error.SetDescription(errorMessage); - error.SetCode(NErrorCode::ERROR); - DestroyQueue(error); -} - -void TDecompressingConsumer::DestroyQueue(const TError& error) { - if (IsDestroyed) { - return; - } - - for (TReadRequestInfo& request : Queue) { - TReadResponse response; - *response.MutableError() = error; - request.Promise.SetValue(TConsumerMessage(std::move(response))); - } - Queue.clear(); - IsDeadPromise.SetValue(error); - IsDestroyed = true; - - DestroyPQLibRef(); -} - -void TDecompressingConsumer::Cancel() { - DestroyQueue(GetCancelReason()); -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h deleted file mode 100644 index c547ef8b3a3..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h +++ /dev/null @@ -1,77 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include "internals.h" -#include "iconsumer_p.h" - -#include <util/generic/maybe.h> - -#include <deque> -#include <memory> -#include <vector> - -namespace NPersQueue { - -class TPQLibPrivate; - -class TDecompressingConsumer: public IConsumerImpl, public std::enable_shared_from_this<TDecompressingConsumer> { -public: - using IConsumerImpl::GetNextMessage; - NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override; - NThreading::TFuture<TError> IsDead() noexcept override; - NThreading::TFuture<void> Destroyed() noexcept override; - void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; - void Commit(const TVector<ui64>& cookies) noexcept override; - void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; - void Init() override; - - ~TDecompressingConsumer(); - TDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); - - void Cancel() override; - -private: - struct TReadRequestInfo { - TReadRequestInfo() = default; - - TReadRequestInfo(const NThreading::TPromise<TConsumerMessage>& promise) - : Promise(promise) - { - } - - NThreading::TPromise<TConsumerMessage> Promise; - NThreading::TFuture<TConsumerMessage> Future; - TMaybe<TConsumerMessage> Data; - - using TMessageFutures = std::vector<NThreading::TFuture<TString>>; - using TBatchFutures = std::vector<TMessageFutures>; - TBatchFutures BatchFutures; - NThreading::TFuture<void> AllDecompressing; - }; - - void ProcessQueue(); - static void SignalProcessQueue(const void* queueTag, std::weak_ptr<TDecompressingConsumer> self, TPQLibPrivate* pqLib); - void DestroyQueue(const TString& errorMessage); - void DestroyQueue(const TError& error); - - template <class T> - void SubscribeForQueueProcessing(NThreading::TFuture<T>& future); - - void RequestDecompressing(TReadRequestInfo& request); - void CopyDataToAnswer(TReadRequestInfo& request); - NThreading::TFuture<TString> RequestDecompressing(const TReadResponse::TData::TMessage& message); - static void Decompress(const TString& data, ECodec codec, NThreading::TPromise<TString>& promise); - void AddNewGetNextMessageRequest(NThreading::TPromise<TConsumerMessage>& promise); - -protected: - TIntrusivePtr<ILogger> Logger; - std::shared_ptr<IConsumerImpl> Subconsumer; - std::deque<TReadRequestInfo> Queue; - TConsumerSettings Settings; - NThreading::TFuture<void> DestroyedFuture; // Subconsumer may be deleted, so store our future here. - NThreading::TPromise<TError> IsDeadPromise = NThreading::NewPromise<TError>(); - bool IsDestroyed = false; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp deleted file mode 100644 index 6c22163e8fa..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#include "decompressing_consumer.h" -#include "local_caller.h" -#include "persqueue_p.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> - -#include <library/cpp/testing/gmock_in_unittest/gmock.h> -#include <library/cpp/testing/unittest/registar.h> - -using namespace testing; - -namespace NPersQueue { - -class TMockConsumer: public IConsumerImpl { -public: - TMockConsumer() - : IConsumerImpl(nullptr, nullptr) - { - } - - NThreading::TFuture<TConsumerCreateResponse> Start(TInstant) noexcept override { - return NThreading::MakeFuture<TConsumerCreateResponse>(MockStart()); - } - - NThreading::TFuture<TError> IsDead() noexcept override { - return MockIsDead(); - } - - NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept override { - return NThreading::MakeFuture<TConsumerMessage>(MockGetNextMessage()); - } - - void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override { - promise.SetValue(MockGetNextMessage()); - } - - void Commit(const TVector<ui64>&) noexcept override { - } - - void RequestPartitionStatus(const TString&, ui64, ui64) noexcept override { - } - - void Cancel() override { - } - - MOCK_METHOD(TConsumerCreateResponse, MockStart, (), ()); - MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ()); - MOCK_METHOD(TConsumerMessage, MockGetNextMessage, (), ()); -}; - -template <class TMock = TMockConsumer> -struct TDecompressingConsumerBootstrap { - ~TDecompressingConsumerBootstrap() { - Lib->CancelObjectsAndWait(); - } - - void Create() { - Lib = new TPQLibPrivate(PQLibSettings); - MockConsumer = std::make_shared<TMock>(); - DecompressingConsumer = std::make_shared<TLocalConsumerImplCaller<TDecompressingConsumer>>(MockConsumer, Settings, Lib->GetSelfRefsAreDeadPtr(), Lib, Logger); - Lib->AddToDestroySet(DecompressingConsumer); - } - - void MakeOKStartResponse() { - TReadResponse rresp; - rresp.MutableInit()->SetSessionId("test-session"); - TConsumerCreateResponse resp(std::move(rresp)); - EXPECT_CALL(*MockConsumer, MockStart()) - .WillOnce(Return(resp)); - } - - void ExpectIsDeadCall() { - EXPECT_CALL(*MockConsumer, MockIsDead()) - .WillOnce(Return(DeadPromise.GetFuture())); - } - - void Start() { - MakeOKStartResponse(); - UNIT_ASSERT_STRINGS_EQUAL(DecompressingConsumer->Start().GetValueSync().Response.GetInit().GetSessionId(), "test-session"); - } - - TConsumerSettings Settings; - TPQLibSettings PQLibSettings; - TIntrusivePtr<TCerrLogger> Logger = new TCerrLogger(TLOG_DEBUG); - TIntrusivePtr<TPQLibPrivate> Lib; - std::shared_ptr<TMock> MockConsumer; - std::shared_ptr<IConsumerImpl> DecompressingConsumer; - NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); -}; - -Y_UNIT_TEST_SUITE(TDecompressingConsumerTest) { - Y_UNIT_TEST(DiesOnDeadSubconsumer) { - TDecompressingConsumerBootstrap<> bootstrap; - bootstrap.Create(); - bootstrap.ExpectIsDeadCall(); - bootstrap.Start(); - - auto isDead = bootstrap.DecompressingConsumer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - bootstrap.DeadPromise.SetValue(TError()); - - isDead.GetValueSync(); // doesn't hang on - } - - Y_UNIT_TEST(PassesNonData) { - TDecompressingConsumerBootstrap<> bootstrap; - bootstrap.Create(); - bootstrap.ExpectIsDeadCall(); - bootstrap.Start(); - - InSequence sequence; - { - TReadResponse resp; - resp.MutableCommit()->AddCookie(42); - - TConsumerMessage ret(std::move(resp)); - - EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) - .WillOnce(Return(ret)); - } - - { - TReadResponse resp; - resp.MutableRelease()->SetTopic("topic!"); - - TConsumerMessage ret(std::move(resp)); - - EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) - .WillOnce(Return(ret)); - } - - auto passed1 = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(passed1.Response.GetCommit().CookieSize(), 1, passed1.Response); - UNIT_ASSERT_VALUES_EQUAL_C(passed1.Response.GetCommit().GetCookie(0), 42, passed1.Response); - - auto passed2 = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync(); - UNIT_ASSERT_STRINGS_EQUAL_C(passed2.Response.GetRelease().GetTopic(), "topic!", passed2.Response); - } - - void ProcessesBrokenChunks(bool skip) { - TDecompressingConsumerBootstrap<> bootstrap; - bootstrap.Settings.SkipBrokenChunks = skip; - bootstrap.Create(); - bootstrap.ExpectIsDeadCall(); - bootstrap.Start(); - - TReadResponse resp; - auto* msg = resp.MutableData()->AddMessageBatch()->AddMessage(); - msg->SetData("hjdhkjhkjhshqsiuhqisuqihsi;"); - msg->MutableMeta()->SetCodec(ECodec::LZOP); - - EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) - .WillOnce(Return(TConsumerMessage(std::move(resp)))); - - auto isDead = bootstrap.DecompressingConsumer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - auto passed = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync(); - - if (skip) { - UNIT_ASSERT(!passed.Response.HasError()); - isDead.HasValue(); - UNIT_ASSERT_VALUES_EQUAL(passed.Response.GetData().MessageBatchSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(passed.Response.GetData().GetMessageBatch(0).MessageSize(), 1); - UNIT_ASSERT(passed.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData().empty()); - } else { - UNIT_ASSERT(passed.Response.HasError()); - isDead.GetValueSync(); - - UNIT_ASSERT(bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync().Response.HasError()); - } - - //DestroyAndWait(bootstrap.DecompressingConsumer); - } - - Y_UNIT_TEST(DiesOnBrokenChunks) { - ProcessesBrokenChunks(false); - } - - Y_UNIT_TEST(SkipsBrokenChunks) { - ProcessesBrokenChunks(true); - } - - static void AddMessage(TReadResponse::TData::TMessageBatch* batch, const TString& sourceData, ECodec codec = ECodec::RAW) { - auto* msg = batch->AddMessage(); - msg->MutableMeta()->SetCodec(codec); - if (codec == ECodec::RAW) { - msg->SetData(sourceData); - } else { - msg->SetData(TData::Encode(sourceData, codec, -1).GetEncodedData()); - } - } - - Y_UNIT_TEST(DecompessesDataInProperChuncksOrder) { - TDecompressingConsumerBootstrap<> bootstrap; - bootstrap.Create(); - bootstrap.ExpectIsDeadCall(); - bootstrap.Start(); - - InSequence sequence; - { - TReadResponse resp; - auto* batch = resp.MutableData()->AddMessageBatch(); - AddMessage(batch, "message1", ECodec::LZOP); - AddMessage(batch, "message2"); - AddMessage(batch, "message3", ECodec::GZIP); - - resp.MutableData()->AddMessageBatch(); - - batch = resp.MutableData()->AddMessageBatch(); - AddMessage(batch, "messageA", ECodec::LZOP); - AddMessage(batch, "messageB", ECodec::ZSTD); - AddMessage(batch, "messageC"); - - EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) - .WillOnce(Return(TConsumerMessage(std::move(resp)))); - } - - { - TReadResponse resp; - AddMessage(resp.MutableData()->AddMessageBatch(), "trololo", ECodec::LZOP); - - EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) - .WillOnce(Return(TConsumerMessage(std::move(resp)))); - } - - auto isDead = bootstrap.DecompressingConsumer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - auto f1 = bootstrap.DecompressingConsumer->GetNextMessage(); - auto f2 = bootstrap.DecompressingConsumer->GetNextMessage(); - - auto data1 = f1.GetValueSync().Response.GetData(); - auto data2 = f2.GetValueSync().Response.GetData(); - - UNIT_ASSERT_VALUES_EQUAL(data1.MessageBatchSize(), 3); - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).MessageSize(), 3); - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(1).MessageSize(), 0); - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).MessageSize(), 3); - - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(0).GetData(), "message1"); - UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW); - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(1).GetData(), "message2"); - UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(1).GetMeta().GetCodec(), ECodec::RAW); - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(2).GetData(), "message3"); - UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(2).GetMeta().GetCodec(), ECodec::RAW); - - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(0).GetData(), "messageA"); - UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW); - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(1).GetData(), "messageB"); - UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(1).GetMeta().GetCodec(), ECodec::RAW); - UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(2).GetData(), "messageC"); - UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(2).GetMeta().GetCodec(), ECodec::RAW); - - - UNIT_ASSERT_VALUES_EQUAL(data2.MessageBatchSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(data2.GetMessageBatch(0).MessageSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(data2.GetMessageBatch(0).GetMessage(0).GetData(), "trololo"); - UNIT_ASSERT_EQUAL(data2.GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW); - - - UNIT_ASSERT(!isDead.HasValue()); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp deleted file mode 100644 index 2ebfcd8d071..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "iconsumer_p.h" -#include "persqueue_p.h" - -namespace NPersQueue { - -TPublicConsumer::TPublicConsumer(std::shared_ptr<IConsumerImpl> impl) - : Impl(std::move(impl)) -{ -} - -TPublicConsumer::~TPublicConsumer() { - Impl->Cancel(); -} - -IConsumerImpl::IConsumerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) - : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib)) -{ -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h deleted file mode 100644 index 35c151b9de5..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h> -#include "interface_common.h" - -namespace NPersQueue { - -class TPQLibPrivate; - -class IConsumerImpl: public IConsumer, public TSyncDestroyed { -public: - IConsumerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); - - virtual void Init() { - } - - using IConsumer::GetNextMessage; - virtual void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept = 0; -}; - -// Consumer that is given to client. -class TPublicConsumer: public IConsumer { -public: - explicit TPublicConsumer(std::shared_ptr<IConsumerImpl> impl); - ~TPublicConsumer(); - - NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override { - return Impl->Start(deadline); - } - - NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept override { - return Impl->GetNextMessage(); - } - - void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override { - Impl->RequestPartitionStatus(topic, partition, generation); - } - - void Commit(const TVector<ui64>& cookies) noexcept override { - Impl->Commit(cookies); - } - - NThreading::TFuture<TError> IsDead() noexcept override { - return Impl->IsDead(); - } - -private: - std::shared_ptr<IConsumerImpl> Impl; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp deleted file mode 100644 index e88c5b12807..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "interface_common.h" -#include "persqueue_p.h" - -namespace NPersQueue { - -const TString& GetCancelReason() -{ - static const TString reason = "Destroyed"; - return reason; -} - -TSyncDestroyed::TSyncDestroyed(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) - : DestroyEventRef(std::move(destroyEventRef)) - , PQLib(std::move(pqLib)) -{ -} - -TSyncDestroyed::~TSyncDestroyed() { - DestroyedPromise.SetValue(); -} - -void TSyncDestroyed::DestroyPQLibRef() { - auto guard = Guard(DestroyLock); - if (DestroyEventRef) { - IsCanceling = true; - PQLib = nullptr; - DestroyEventRef = nullptr; - } -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h deleted file mode 100644 index bd3a228b2c0..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include <library/cpp/threading/future/future.h> - -#include <util/generic/ptr.h> -#include <util/system/spinlock.h> - -#include <memory> - -namespace NPersQueue { - -class TPQLibPrivate; - -const TString& GetCancelReason(); - -class TSyncDestroyed { -protected: - TSyncDestroyed(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); - -public: - // Get future that is signalled after object's destructor. - // Non-pqlib threads can wait on this future to ensure that - // all async operations are finished. - virtual NThreading::TFuture<void> Destroyed() noexcept { - return DestroyedPromise.GetFuture(); - } - - // This destructor will be executed after real object's destructor - // in which it is expected to set all owned promises. - virtual ~TSyncDestroyed(); - - virtual void Cancel() = 0; - - void SetDestroyEventRef(std::shared_ptr<void> ref) { - DestroyEventRef = std::move(ref); - } - -protected: - // Needed for proper PQLib deinitialization - void DestroyPQLibRef(); - -protected: - NThreading::TPromise<void> DestroyedPromise = NThreading::NewPromise<void>(); - std::shared_ptr<void> DestroyEventRef; - TIntrusivePtr<TPQLibPrivate> PQLib; - bool IsCanceling = false; - TAdaptiveLock DestroyLock; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h deleted file mode 100644 index 5c893da75e7..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h +++ /dev/null @@ -1,105 +0,0 @@ -#pragma once -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> - -#include <library/cpp/grpc/common/time_point.h> -#include <library/cpp/threading/future/future.h> - -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/string/cast.h> - -#include <grpc++/channel.h> -#include <grpc++/create_channel.h> - -#include <contrib/libs/grpc/include/grpcpp/impl/codegen/client_context.h> - -#include <chrono> - -#define WRITE_LOG(msg, srcId, sessionId, level, logger) \ - if (logger && logger->IsEnabled(level)) { \ - logger->Log(TStringBuilder() << msg, srcId, sessionId, level); \ - } - -#define DEBUG_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_DEBUG, Logger) -#define INFO_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_INFO, Logger) -#define WARN_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_WARNING, Logger) -#define ERR_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_ERR, Logger) - -namespace NPersQueue { - -TString GetToken(ICredentialsProvider* credentials); - -void FillMetaHeaders(grpc::ClientContext& context, const TString& database, ICredentialsProvider* credentials); - -bool UseCDS(const TServerSetting& server); - -struct TWriteData { - TProducerSeqNo SeqNo; - TData Data; - - TWriteData(ui64 seqNo, TData&& data) - : SeqNo(seqNo) - , Data(std::move(data)) - {} -}; - -class IQueueEvent { -public: - virtual ~IQueueEvent() = default; - - //! Execute an action defined by implementation. - virtual bool Execute(bool ok) = 0; - - //! Finish and destroy request. - virtual void DestroyRequest() = 0; -}; - -class IHandler : public TAtomicRefCount<IHandler> { -public: - IHandler() - {} - - virtual ~IHandler() - {} - - virtual void Destroy(const TError&) = 0; - virtual void Done() = 0; - virtual TString ToString() = 0; -}; - -using IHandlerPtr = TIntrusivePtr<IHandler>; - -class TQueueEvent : public IQueueEvent { -public: - TQueueEvent(IHandlerPtr handler) - : Handler(std::move(handler)) - { - Y_ASSERT(Handler); - } - -private: - bool Execute(bool ok) override { - if (ok) { - Handler->Done(); - } else { - TError error; - error.SetDescription("event " + Handler->ToString() + " failed"); - error.SetCode(NErrorCode::ERROR); - Handler->Destroy(error); - } - return false; - } - - virtual ~TQueueEvent() { - Handler.Reset(); - } - - void DestroyRequest() override - { - delete this; - } - - IHandlerPtr Handler; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp deleted file mode 100644 index f245df74fec..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "iprocessor_p.h" -#include "persqueue_p.h" - -namespace NPersQueue { - -TPublicProcessor::TPublicProcessor(std::shared_ptr<IProcessorImpl> impl) - : Impl(std::move(impl)) -{ -} - -TPublicProcessor::~TPublicProcessor() { - Impl->Cancel(); -} - -IProcessorImpl::IProcessorImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) - : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib)) -{ -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h deleted file mode 100644 index e315b8412f9..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> -#include "interface_common.h" - -namespace NPersQueue { - -class TPQLibPrivate; - -struct IProcessorImpl: public IProcessor, public TSyncDestroyed { - using IProcessor::GetNextData; - - IProcessorImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); - - // Initialization after constructor (for example, for correct call of shared_from_this()) - virtual void Init() { - } - virtual void GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept = 0; -}; - -// Processor that is given to client. -class TPublicProcessor: public IProcessor { -public: - explicit TPublicProcessor(std::shared_ptr<IProcessorImpl> impl); - ~TPublicProcessor(); - - NThreading::TFuture<TOriginData> GetNextData() noexcept override { - return Impl->GetNextData(); - } - -private: - std::shared_ptr<IProcessorImpl> Impl; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp deleted file mode 100644 index c7a267b862b..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "iproducer_p.h" -#include "persqueue_p.h" - -namespace NPersQueue { - -TPublicProducer::TPublicProducer(std::shared_ptr<IProducerImpl> impl) - : Impl(std::move(impl)) -{ -} - -TPublicProducer::~TPublicProducer() { - Impl->Cancel(); -} - -IProducerImpl::IProducerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) - : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib)) -{ -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h deleted file mode 100644 index 437798c5515..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> -#include "interface_common.h" - -namespace NPersQueue { - -class TPQLibPrivate; - -struct IProducerImpl: public IProducer, public TSyncDestroyed { - using IProducer::Write; - - IProducerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); - - // Initialization after constructor (for example, for correct call of shared_from_this()) - virtual void Init() { - } - virtual void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept = 0; - virtual void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept = 0; - - NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo, TData) noexcept override { - Y_FAIL(""); - return NThreading::NewPromise<TProducerCommitResponse>().GetFuture(); - } - - NThreading::TFuture<TProducerCommitResponse> Write(TData) noexcept override { - Y_FAIL(""); - return NThreading::NewPromise<TProducerCommitResponse>().GetFuture(); - } -}; - -// Producer that is given to client. -class TPublicProducer: public IProducer { -public: - explicit TPublicProducer(std::shared_ptr<IProducerImpl> impl); - ~TPublicProducer(); - - NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept { - return Impl->Start(deadline); - } - - NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept { - return Impl->Write(seqNo, std::move(data)); - } - - NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept { - return Impl->Write(std::move(data)); - } - - NThreading::TFuture<TError> IsDead() noexcept { - return Impl->IsDead(); - } - -private: - std::shared_ptr<IProducerImpl> Impl; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h deleted file mode 100644 index 2ac0b3ec18a..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h +++ /dev/null @@ -1,303 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include "interface_common.h" -#include "internals.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include "persqueue_p.h" - -namespace NPersQueue { - -template <class TImpl> -class TCancelCaller: public TImpl { -public: - using TImpl::TImpl; - - std::shared_ptr<TCancelCaller> MakeSharedRef() { - return std::static_pointer_cast<TCancelCaller>(this->shared_from_this()); - } - - void Cancel() override { - auto guard = Guard(this->DestroyLock); - if (!this->IsCanceling) { - this->IsCanceling = true; - AddCancelCaller(); - } - } - -protected: - const TImpl* QueueTag() const { // we must have exactly the same thread tag as TImpl object - return this; - } - - static void MakeCanceledError(TError& error) { - error.SetDescription(GetCancelReason()); - error.SetCode(NErrorCode::ERROR); - } - - template <class TResponseProto> - static void MakeCanceledError(TResponseProto& resp) { - MakeCanceledError(*resp.MutableError()); - } - - template <class TResponse, class... TArgs> - static void MakeCanceledError(NThreading::TPromise<TResponse>& promise, TArgs&&...) { // the most common case for calls - decltype(promise.GetValue().Response) responseProto; - MakeCanceledError(responseProto); - promise.SetValue(TResponse(std::move(responseProto))); - } - - template <class TResponse> - static TResponse MakeCanceledError() { - TResponse resp; - MakeCanceledError(*resp.MutableError()); - return resp; - } - -private: - void AddCancelCaller() { - std::weak_ptr<TCancelCaller> selfWeak = MakeSharedRef(); - auto caller = [selfWeak]() mutable { - auto selfShared = selfWeak.lock(); - if (selfShared) { - selfShared->TImpl::Cancel(); - } - }; - Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(QueueTag()).AddFunc(caller)); - } -}; - -template <class TImpl, class TStartResponse> -class TLocalStartDeadImplCaller: public TCancelCaller<TImpl> { -protected: - using TCancelCaller<TImpl>::TCancelCaller; - - NThreading::TFuture<TStartResponse> Start(TInstant deadline) noexcept override { - IsDead(); // subscribe for impl future - auto guard = Guard(this->DestroyLock); - if (this->IsCanceling) { - guard.Release(); - decltype(reinterpret_cast<TStartResponse*>(0)->Response) respProto; - this->MakeCanceledError(respProto); - return NThreading::MakeFuture(TStartResponse(std::move(respProto))); - } - if (StartPromise.Initialized()) { - guard.Release(); - decltype(reinterpret_cast<TStartResponse*>(0)->Response) respProto; - TError& error = *respProto.MutableError(); - error.SetDescription("Start was already called"); - error.SetCode(NErrorCode::BAD_REQUEST); - return NThreading::MakeFuture(TStartResponse(std::move(respProto))); - } - StartPromise = NThreading::NewPromise<TStartResponse>(); - AddStartCaller(deadline); - return StartPromise.GetFuture(); - } - - NThreading::TFuture<TError> IsDead() noexcept override { - auto guard = Guard(this->DestroyLock); - if (this->IsCanceling && !IsDeadCalled) { - guard.Release(); - TError err; - this->MakeCanceledError(err); - return NThreading::MakeFuture(err); - } - if (!IsDeadCalled) { - AddIsDeadCaller(); - } - return DeadPromise.GetFuture(); - } - -private: - std::shared_ptr<TLocalStartDeadImplCaller> MakeSharedRef() { - return std::static_pointer_cast<TLocalStartDeadImplCaller>(this->shared_from_this()); - } - - void AddIsDeadCaller() { - IsDeadCalled = true; - std::shared_ptr<TLocalStartDeadImplCaller> selfShared = MakeSharedRef(); - auto isDeadCaller = [selfShared]() mutable { - selfShared->CallIsDead(); - }; - Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(isDeadCaller)); - } - - void CallIsDead() { - auto deadPromise = DeadPromise; - TImpl::IsDead().Subscribe([deadPromise](const auto& future) mutable { - deadPromise.SetValue(future.GetValue()); - }); - } - - void AddStartCaller(TInstant deadline) { - std::shared_ptr<TLocalStartDeadImplCaller> selfShared = MakeSharedRef(); - auto startCaller = [selfShared, deadline]() mutable { - selfShared->CallStart(deadline); - }; - Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(startCaller)); - } - - void CallStart(TInstant deadline) { - auto startPromise = StartPromise; - TImpl::Start(deadline).Subscribe([startPromise](const auto& future) mutable { - startPromise.SetValue(future.GetValue()); - }); - } - -private: - NThreading::TPromise<TStartResponse> StartPromise; - NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); - bool IsDeadCalled = false; -}; - -#define LOCAL_CALLER(Func, Args, DeclTail, ...) \ - void Func Args DeclTail { \ - auto guard = Guard(this->DestroyLock); \ - if (this->IsCanceling) { \ - guard.Release(); \ - this->MakeCanceledError(__VA_ARGS__); \ - return; \ - } \ - auto selfShared = MakeSharedRef(); \ - auto caller = [selfShared, __VA_ARGS__]() mutable { \ - selfShared->TImpl::Func(__VA_ARGS__); \ - }; \ - Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(caller));\ - } - -// Assumes that there is function with promise interface -// first arg must be promise -#define LOCAL_CALLER_WITH_FUTURE(ReturnType, Func, Args, DeclTail, ...) \ - NThreading::TFuture<ReturnType> Func Args DeclTail { \ - auto promise = NThreading::NewPromise<ReturnType>(); \ - Func(__VA_ARGS__); \ - return promise.GetFuture(); \ - } - -// Helper classes for producer/consumer/processor impls. -// Wrappers that delegate calls to its methods to proper thread pool local threads. -// The result is that all calls to impl are serialized and locks are no more needed. -// -// Implies: -// That impl class is std::enable_shared_from_this descendant. - -template <class TImpl> -class TLocalProducerImplCaller: public TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse> { -public: - using TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse>::TLocalStartDeadImplCaller; - - LOCAL_CALLER(Write, - (NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data), - noexcept override, - promise, seqNo, data) - - LOCAL_CALLER(Write, - (NThreading::TPromise<TProducerCommitResponse>& promise, TData data), - noexcept override, - promise, data) - - LOCAL_CALLER_WITH_FUTURE(TProducerCommitResponse, - Write, - (TProducerSeqNo seqNo, TData data), - noexcept override, - promise, seqNo, data) - - LOCAL_CALLER_WITH_FUTURE(TProducerCommitResponse, - Write, - (TData data), - noexcept override, - promise, data) - -protected: - using TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse>::MakeCanceledError; - static void MakeCanceledError(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) { - TWriteResponse responseProto; - MakeCanceledError(responseProto); - promise.SetValue(TProducerCommitResponse(seqNo, std::move(data), std::move(responseProto))); - } - - static void MakeCanceledError(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) { - MakeCanceledError(promise, TProducerSeqNo(0), std::move(data)); - } - -private: - std::shared_ptr<TLocalProducerImplCaller> MakeSharedRef() { - return std::static_pointer_cast<TLocalProducerImplCaller>(this->shared_from_this()); - } -}; - -template <class TImpl> -class TLocalConsumerImplCaller: public TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse> { -public: - using TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse>::TLocalStartDeadImplCaller; - - LOCAL_CALLER(GetNextMessage, - (NThreading::TPromise<TConsumerMessage>& promise), - noexcept override, - promise) - - LOCAL_CALLER_WITH_FUTURE(TConsumerMessage, - GetNextMessage, - (), - noexcept override, - promise) - - LOCAL_CALLER(Commit, - (const TVector<ui64>& cookies), - noexcept override, - cookies) - - LOCAL_CALLER(RequestPartitionStatus, - (const TString& topic, ui64 partition, ui64 generation), - noexcept override, - topic, partition, generation) - -private: - using TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse>::MakeCanceledError; - static void MakeCanceledError(const TVector<ui64>& cookies) { - Y_UNUSED(cookies); - // Do nothing, because this method doesn't return future - } - - static void MakeCanceledError(const TString&, ui64, ui64) { - // Do nothing, because this method doesn't return future - } - - - std::shared_ptr<TLocalConsumerImplCaller> MakeSharedRef() { - return std::static_pointer_cast<TLocalConsumerImplCaller>(this->shared_from_this()); - } -}; - -template <class TImpl> -class TLocalProcessorImplCaller: public TCancelCaller<TImpl> { -public: - using TCancelCaller<TImpl>::TCancelCaller; - - LOCAL_CALLER(GetNextData, - (NThreading::TPromise<TOriginData>& promise), - noexcept override, - promise) - - LOCAL_CALLER_WITH_FUTURE(TOriginData, - GetNextData, - (), - noexcept override, - promise) - -private: - using TCancelCaller<TImpl>::MakeCanceledError; - static void MakeCanceledError(NThreading::TPromise<TOriginData>& promise) { - promise.SetValue(TOriginData()); // with empty data - } - - std::shared_ptr<TLocalProcessorImplCaller> MakeSharedRef() { - return std::static_pointer_cast<TLocalProcessorImplCaller>(this->shared_from_this()); - } -}; - -#undef LOCAL_CALLER -#undef LOCAL_CALLER_WITH_FUTURE - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp deleted file mode 100644 index 98d44d14c5f..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h> -#include <util/datetime/base.h> -#include <util/string/builder.h> -#include <util/stream/str.h> - -namespace NPersQueue { - -static const TStringBuf LogLevelsStrings[] = { - "EMERG", - "ALERT", - "CRITICAL_INFO", - "ERROR", - "WARNING", - "NOTICE", - "INFO", - "DEBUG", -}; - -TStringBuf TCerrLogger::LevelToString(int level) { - return LogLevelsStrings[ClampVal(level, 0, int(Y_ARRAY_SIZE(LogLevelsStrings) - 1))]; -} - -void TCerrLogger::Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) { - if (level > Level) { - return; - } - - TStringBuilder message; - message << TInstant::Now() << " :" << LevelToString(level) << ":"; - if (sourceId) { - message << " SourceId [" << sourceId << "]:"; - } - if (sessionId) { - message << " SessionId [" << sessionId << "]:"; - } - message << " " << msg << "\n"; - Cerr << message; -} - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp deleted file mode 100644 index cd986532dc4..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp +++ /dev/null @@ -1,645 +0,0 @@ -#include "multicluster_consumer.h" -#include "persqueue_p.h" - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -#include <library/cpp/containers/intrusive_rb_tree/rb_tree.h> - -#include <util/generic/guid.h> -#include <util/generic/strbuf.h> - -namespace NPersQueue { - -struct TMultiClusterConsumer::TSubconsumerInfo { - TSubconsumerInfo(std::shared_ptr<IConsumerImpl> consumer); - - std::shared_ptr<IConsumerImpl> Consumer; - NThreading::TFuture<TConsumerCreateResponse> StartFuture; - NThreading::TFuture<TError> DeadFuture; - bool StartResponseWasGot = false; // We call methods of this consumer only after this flag was set. - - std::deque<NThreading::TFuture<TConsumerMessage>> PendingRequests; -}; - -namespace { - -struct TConsumerCookieKey; -struct TUserCookieMappingItem; -struct TConsumerCookieMappingItem; - -struct TConsumerCookieKey { - size_t Subconsumer = 0; - ui64 ConsumerCookie = 0; - - TConsumerCookieKey(size_t subconsumer, ui64 consumerCookie) - : Subconsumer(subconsumer) - , ConsumerCookie(consumerCookie) - { - } - - bool operator<(const TConsumerCookieKey& other) const { - return std::make_pair(Subconsumer, ConsumerCookie) < std::make_pair(other.Subconsumer, other.ConsumerCookie); - } -}; - -struct TUserCookieCmp { - static ui64 GetKey(const TUserCookieMappingItem& item); - - static bool Compare(ui64 l, const TUserCookieMappingItem& r) { - return l < GetKey(r); - } - - static bool Compare(const TUserCookieMappingItem& l, ui64 r) { - return GetKey(l) < r; - } - - static bool Compare(const TUserCookieMappingItem& l, const TUserCookieMappingItem& r) { - return GetKey(l) < GetKey(r); - } -}; - -struct TConsumerCookieCmp { - static TConsumerCookieKey GetKey(const TConsumerCookieMappingItem& item); - - static bool Compare(const TConsumerCookieKey& l, const TConsumerCookieMappingItem& r) { - return l < GetKey(r); - } - - static bool Compare(const TConsumerCookieMappingItem& l, const TConsumerCookieKey& r) { - return GetKey(l) < r; - } - - static bool Compare(const TConsumerCookieMappingItem& l, const TConsumerCookieMappingItem& r) { - return GetKey(l) < GetKey(r); - } -}; - -struct TUserCookieMappingItem : public TRbTreeItem<TUserCookieMappingItem, TUserCookieCmp> { -}; - -struct TConsumerCookieMappingItem : public TRbTreeItem<TConsumerCookieMappingItem, TConsumerCookieCmp> { -}; - -bool IsRetryable(const grpc::Status& status) { - switch (status.error_code()) { - case grpc::OK: - case grpc::CANCELLED: - case grpc::INVALID_ARGUMENT: - case grpc::NOT_FOUND: - case grpc::ALREADY_EXISTS: - case grpc::PERMISSION_DENIED: - case grpc::UNAUTHENTICATED: - case grpc::FAILED_PRECONDITION: - case grpc::ABORTED: - case grpc::OUT_OF_RANGE: - case grpc::UNIMPLEMENTED: - return false; - - case grpc::UNKNOWN: - case grpc::DEADLINE_EXCEEDED: - case grpc::RESOURCE_EXHAUSTED: - case grpc::INTERNAL: - case grpc::UNAVAILABLE: - case grpc::DATA_LOSS: - case grpc::DO_NOT_USE: - return true; - } -} - -} // namespace - -struct TMultiClusterConsumer::TCookieMappingItem : public TUserCookieMappingItem, public TConsumerCookieMappingItem { - TCookieMappingItem(size_t subconsumer, ui64 consumerCookie, ui64 userCookie) - : Subconsumer(subconsumer) - , ConsumerCookie(consumerCookie) - , UserCookie(userCookie) - { - } - - size_t Subconsumer = 0; - ui64 ConsumerCookie = 0; - ui64 UserCookie = 0; -}; - -namespace { - -ui64 TUserCookieCmp::GetKey(const TUserCookieMappingItem& item) { - return static_cast<const TMultiClusterConsumer::TCookieMappingItem&>(item).UserCookie; -} - -TConsumerCookieKey TConsumerCookieCmp::GetKey(const TConsumerCookieMappingItem& item) { - const auto& src = static_cast<const TMultiClusterConsumer::TCookieMappingItem&>(item); - return { src.Subconsumer, src.ConsumerCookie }; -} - -} // namespace - -struct TMultiClusterConsumer::TCookieMapping { - using TUserCookieTree = TRbTree<TUserCookieMappingItem, TUserCookieCmp>; - using TConsumerCookieTree = TRbTree<TConsumerCookieMappingItem, TConsumerCookieCmp>; - - struct TConsumerCookieDestroy : public TConsumerCookieTree::TDestroy { - void operator()(TConsumerCookieMappingItem& item) const { - TDestroy::operator()(item); // Remove from tree - delete static_cast<TCookieMappingItem*>(&item); - } - }; - - ~TCookieMapping() { - UserCookieTree.ForEachNoOrder(TUserCookieTree::TDestroy()); - UserCookieTree.Init(); - - ConsumerCookieTree.ForEachNoOrder(TConsumerCookieDestroy()); - ConsumerCookieTree.Init(); - } - - ui64 AddMapping(size_t subconsumer, ui64 cookie) { // Returns user cookie - auto* newItem = new TCookieMappingItem(subconsumer, cookie, NextCookie++); - UserCookieTree.Insert(newItem); - ConsumerCookieTree.Insert(newItem); - return newItem->UserCookie; - } - - TCookieMappingItem* FindMapping(ui64 userCookie) { - return DownCast(UserCookieTree.Find(userCookie)); - } - - TCookieMappingItem* FindMapping(size_t subconsumer, ui64 cookie) { - return DownCast(ConsumerCookieTree.Find(TConsumerCookieKey(subconsumer, cookie))); - } - - void RemoveUserCookieMapping(TCookieMappingItem* item) { - Y_ASSERT(item); - static_cast<TUserCookieMappingItem*>(item)->UnLink(); - } - - void RemoveMapping(TCookieMappingItem* item) { - Y_ASSERT(item); - Y_ASSERT(!static_cast<TUserCookieMappingItem*>(item)->ParentTree()); - static_cast<TUserCookieMappingItem*>(item)->UnLink(); // Just in case - static_cast<TConsumerCookieMappingItem*>(item)->UnLink(); - delete item; - } - -private: - template <class T> - static TCookieMappingItem* DownCast(T* item) { - if (item) { - return static_cast<TCookieMappingItem*>(item); - } else { - return nullptr; - } - } - - TUserCookieTree UserCookieTree; - TConsumerCookieTree ConsumerCookieTree; - ui64 NextCookie = 1; -}; - -TMultiClusterConsumer::TMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) - : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) - , Settings(settings) - , Logger(std::move(logger)) - , SessionId(CreateGuidAsString()) - , CookieMapping(MakeHolder<TCookieMapping>()) -{ - PatchSettings(); -} - -TMultiClusterConsumer::~TMultiClusterConsumer() { - Destroy(); -} - -void TMultiClusterConsumer::PatchSettings() { - if (!Settings.ReconnectOnFailure) { - WARN_LOG("Ignoring ReconnectOnFailure=false option for multicluster consumer", "", SessionId); - Settings.ReconnectOnFailure = true; - } - if (Settings.MaxAttempts != std::numeric_limits<unsigned>::max()) { - WARN_LOG("Ignoring MaxAttempts option for multicluster consumer", "", SessionId); - Settings.MaxAttempts = std::numeric_limits<unsigned>::max(); - } -} - -NThreading::TFuture<TConsumerCreateResponse> TMultiClusterConsumer::Start(TInstant deadline) noexcept { - Y_VERIFY(State == EState::Created); - StartClusterDiscovery(deadline); - return StartPromise.GetFuture(); -} - -NThreading::TFuture<TError> TMultiClusterConsumer::IsDead() noexcept { - return IsDeadPromise.GetFuture(); -} - -void TMultiClusterConsumer::ScheduleClusterDiscoveryRetry(TInstant deadline) { - const TInstant now = TInstant::Now(); - if (ClusterDiscoveryAttemptsDone >= Settings.MaxAttempts || now >= deadline) { - Destroy(TStringBuilder() << "Failed " << ClusterDiscoveryAttemptsDone << " cluster discovery attempts", NErrorCode::CREATE_TIMEOUT); - return; - } - - if (!IsRetryable(ClusterDiscoverResult->first)) { - Destroy(TStringBuilder() << "Cluster discovery failed: " << static_cast<int>(ClusterDiscoverResult->first.error_code())); - return; - } - - TDuration delay = Min(Settings.MaxReconnectionDelay, ClusterDiscoveryAttemptsDone * Settings.ReconnectionDelay, deadline - now); - std::weak_ptr<TMultiClusterConsumer> weakRef = shared_from_this(); - PQLib->GetScheduler().Schedule(delay, this, [weakRef, deadline]() { - if (auto self = weakRef.lock()) { - self->StartClusterDiscovery(deadline); - } - }); -} - -void TMultiClusterConsumer::StartClusterDiscovery(TInstant deadline) { - if (State == EState::Dead) { - return; - } - - State = EState::WaitingClusterDiscovery; - ++ClusterDiscoveryAttemptsDone; - DEBUG_LOG("Starting cluster discovery", "", SessionId); - - auto discoverer = MakeIntrusive<TConsumerChannelOverCdsImpl>(Settings, PQLib.Get(), Logger); - discoverer->Start(); - ClusterDiscoverResult = discoverer->GetResultPtr(); - ClusterDiscoverer = std::move(discoverer); - - std::weak_ptr<TMultiClusterConsumer> self = shared_from_this(); - auto handler = [self, deadline](const auto&) { - if (auto selfShared = self.lock()) { - selfShared->OnClusterDiscoveryDone(deadline); - } - }; - PQLib->Subscribe(ClusterDiscoverer->GetChannel(), this, handler); -} - -void TMultiClusterConsumer::OnClusterDiscoveryDone(TInstant deadline) { - if (State != EState::WaitingClusterDiscovery) { - return; - } - - ClusterDiscoverer = nullptr; // delete - if (!ClusterDiscoverResult->first.ok()) { - INFO_LOG("Failed to discover clusters. Grpc error: " << static_cast<int>(ClusterDiscoverResult->first.error_code()), "", SessionId); - ScheduleClusterDiscoveryRetry(deadline); // Destroys if we shouldn't retry. - return; - } - - if (static_cast<size_t>(ClusterDiscoverResult->second.read_sessions_clusters_size()) != Settings.Topics.size()) { - Destroy("Got unexpected cluster discovery result"); - return; - } - - if (deadline != TInstant::Max()) { - const TInstant now = TInstant::Now(); - if (now > deadline) { - Destroy("Start timeout", NErrorCode::CREATE_TIMEOUT); - return; - } - - std::weak_ptr<TMultiClusterConsumer> self = shared_from_this(); - auto onStartTimeout = [self] { - if (auto selfShared = self.lock()) { - selfShared->OnStartTimeout(); - } - }; - StartDeadlineCallback = - PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout); - } - - State = EState::StartingSubconsumers; - DEBUG_LOG("Starting subconsumers", "", SessionId); - - // Group topics by clusters. - THashMap<TString, TVector<TString>> clusterToTopics; - for (size_t topicIndex = 0, topicsCount = Settings.Topics.size(); topicIndex < topicsCount; ++topicIndex) { - const Ydb::PersQueue::ClusterDiscovery::ReadSessionClusters& clusters = ClusterDiscoverResult->second.read_sessions_clusters(topicIndex); - const TString& topic = Settings.Topics[topicIndex]; - for (const Ydb::PersQueue::ClusterDiscovery::ClusterInfo& clusterInfo : clusters.clusters()) { - clusterToTopics[clusterInfo.endpoint()].push_back(topic); - } - } - - if (clusterToTopics.empty()) { - Destroy("Got empty endpoint set from cluster discovery"); - return; - } - - // Start consumer on every cluster with its topics set. - for (auto&& [clusterEndpoint, topics] : clusterToTopics) { - // Settings. - TConsumerSettings settings = Settings; - settings.Server = ApplyClusterEndpoint(settings.Server, clusterEndpoint); - settings.ReadFromAllClusterSources = false; - settings.Topics = std::move(topics); - settings.Unpack = false; // Unpack is being done in upper level decompressing consumer. - settings.MaxMemoryUsage = Max(settings.MaxMemoryUsage / clusterToTopics.size(), static_cast<size_t>(1)); - if (settings.MaxUncommittedSize > 0) { // Limit is enabled. - settings.MaxUncommittedSize = Max(settings.MaxUncommittedSize / clusterToTopics.size(), static_cast<size_t>(1)); - } - if (settings.MaxUncommittedCount > 0) { // Limit is enabled. - settings.MaxUncommittedCount = Max(settings.MaxUncommittedCount / clusterToTopics.size(), static_cast<size_t>(1)); - } - const size_t subconsumerIndex = Subconsumers.size(); - - // Create subconsumer. - Subconsumers.push_back(PQLib->CreateRawRetryingConsumer(settings, DestroyEventRef, Logger)); - - // Subscribe on start. - std::weak_ptr<TMultiClusterConsumer> self = shared_from_this(); - auto handler = [self, subconsumerIndex](const auto&) { - if (auto selfShared = self.lock()) { - selfShared->OnSubconsumerStarted(subconsumerIndex); - } - }; - PQLib->Subscribe(Subconsumers.back().StartFuture, this, handler); - } -} - -void TMultiClusterConsumer::OnStartTimeout() { - if (State == EState::Dead || State == EState::Working) { - return; - } - - StartDeadlineCallback = nullptr; - Destroy("Start timeout", NErrorCode::CREATE_TIMEOUT); -} - - -void TMultiClusterConsumer::OnSubconsumerStarted(size_t subconsumerIndex) { - if (State == EState::Dead) { - return; - } - - auto& subconsumerInfo = Subconsumers[subconsumerIndex]; - const auto& result = subconsumerInfo.StartFuture.GetValue(); - if (result.Response.HasError()) { - WARN_LOG("Got error on starting subconsumer: " << result.Response.GetError(), "", SessionId); - Destroy(result.Response.GetError()); - return; - } - - // Process start response - subconsumerInfo.StartResponseWasGot = true; - INFO_LOG("Subconsumer " << subconsumerIndex << " started session with id " << result.Response.GetInit().GetSessionId(), "", SessionId); - - // Move to new state - if (State == EState::StartingSubconsumers) { - State = EState::Working; - if (StartDeadlineCallback) { - StartDeadlineCallback->TryCancel(); - StartDeadlineCallback = nullptr; - } - - { - TReadResponse resp; - resp.MutableInit()->SetSessionId(SessionId); - StartPromise.SetValue(TConsumerCreateResponse(std::move(resp))); - } - } else { - RequestSubconsumers(); // Make requests for new subconsumer. - } -} - -void TMultiClusterConsumer::Commit(const TVector<ui64>& cookies) noexcept { - if (State != EState::Working) { - Destroy("Requesting commit, but consumer is not in working state", NErrorCode::BAD_REQUEST); - return; - } - - TVector<TVector<ui64>> subconsumersCookies(Subconsumers.size()); - for (ui64 userCookie : cookies) { - TCookieMappingItem* mapping = CookieMapping->FindMapping(userCookie); - if (!mapping) { - Destroy(TStringBuilder() << "Wrong cookie " << userCookie, NErrorCode::WRONG_COOKIE); - return; - } - Y_ASSERT(mapping->Subconsumer < Subconsumers.size()); - subconsumersCookies[mapping->Subconsumer].push_back(mapping->ConsumerCookie); - CookieMapping->RemoveUserCookieMapping(mapping); // Avoid double commit. This will ensure error on second commit with the same cookie. - } - - for (size_t subconsumerIndex = 0; subconsumerIndex < Subconsumers.size(); ++subconsumerIndex) { - const TVector<ui64>& subconsumerCommitRequest = subconsumersCookies[subconsumerIndex]; - if (!subconsumerCommitRequest.empty()) { - Subconsumers[subconsumerIndex].Consumer->Commit(subconsumerCommitRequest); - } - } -} - -void TMultiClusterConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { - if (State != EState::Working) { - promise.SetValue(MakeResponse(MakeError("Requesting next message, but consumer is not in working state", NErrorCode::BAD_REQUEST))); - return; - } - - Requests.push_back(promise); - CheckReadyResponses(); - RequestSubconsumers(); -} - -void TMultiClusterConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { - if (State != EState::Working) { - Destroy("Requesting partition status, but consumer is not in working state", NErrorCode::BAD_REQUEST); - return; - } - - const auto subconsumerIdIt = OldTopicName2Subconsumer.find(topic); - if (subconsumerIdIt != OldTopicName2Subconsumer.end()) { - Y_ASSERT(subconsumerIdIt->second < Subconsumers.size()); - Subconsumers[subconsumerIdIt->second].Consumer->RequestPartitionStatus(topic, partition, generation); - } else { - WARN_LOG("Requested partition status for topic \"" << topic << "\" (partition " << partition << ", generation " << generation << "), but there is no such lock session. Ignoring request", "", SessionId); - } -} - -void TMultiClusterConsumer::CheckReadyResponses() { - const size_t prevCurrentSubconsumer = CurrentSubconsumer; - - do { - if (!ProcessReadyResponses(CurrentSubconsumer)) { - break; - } - - // Next subconsumer in round robin way. - ++CurrentSubconsumer; - if (CurrentSubconsumer == Subconsumers.size()) { - CurrentSubconsumer = 0; - } - } while (CurrentSubconsumer != prevCurrentSubconsumer && !Requests.empty()); -} - -bool TMultiClusterConsumer::ProcessReadyResponses(size_t subconsumerIndex) { - if (Subconsumers.empty()) { - Y_VERIFY(State == EState::Dead); - return false; - } - Y_VERIFY(subconsumerIndex < Subconsumers.size()); - TSubconsumerInfo& consumerInfo = Subconsumers[subconsumerIndex]; - while (!consumerInfo.PendingRequests.empty() && !Requests.empty() && consumerInfo.PendingRequests.front().HasValue()) { - if (!TranslateConsumerMessage(Requests.front(), consumerInfo.PendingRequests.front().ExtractValue(), subconsumerIndex)) { - return false; - } - Requests.pop_front(); - consumerInfo.PendingRequests.pop_front(); - } - return true; -} - -bool TMultiClusterConsumer::TranslateConsumerMessage(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { - switch (subconsumerResponse.Type) { - case EMT_LOCK: - return TranslateConsumerMessageLock(promise, std::move(subconsumerResponse), subconsumerIndex); - case EMT_RELEASE: - return TranslateConsumerMessageRelease(promise, std::move(subconsumerResponse), subconsumerIndex); - case EMT_DATA: - return TranslateConsumerMessageData(promise, std::move(subconsumerResponse), subconsumerIndex); - case EMT_ERROR: - return TranslateConsumerMessageError(promise, std::move(subconsumerResponse), subconsumerIndex); - case EMT_STATUS: - return TranslateConsumerMessageStatus(promise, std::move(subconsumerResponse), subconsumerIndex); - case EMT_COMMIT: - return TranslateConsumerMessageCommit(promise, std::move(subconsumerResponse), subconsumerIndex); - } -} - -bool TMultiClusterConsumer::TranslateConsumerMessageData(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { - auto* data = subconsumerResponse.Response.MutableData(); - data->SetCookie(CookieMapping->AddMapping(subconsumerIndex, data->GetCookie())); - promise.SetValue(std::move(subconsumerResponse)); - return true; -} - -bool TMultiClusterConsumer::TranslateConsumerMessageCommit(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { - for (ui64& consumerCookie : *subconsumerResponse.Response.MutableCommit()->MutableCookie()) { - auto* mapping = CookieMapping->FindMapping(subconsumerIndex, consumerCookie); - if (!mapping) { - Destroy(TStringBuilder() << "Received unknown cookie " << consumerCookie << " commit"); - return false; - } - Y_VERIFY(mapping); - consumerCookie = mapping->UserCookie; - CookieMapping->RemoveMapping(mapping); // Invalidates mapping - } - promise.SetValue(std::move(subconsumerResponse)); - return true; -} - -bool TMultiClusterConsumer::TranslateConsumerMessageLock(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { - OldTopicName2Subconsumer[subconsumerResponse.Response.GetLock().GetTopic()] = subconsumerIndex; - promise.SetValue(std::move(subconsumerResponse)); - return true; -} - -bool TMultiClusterConsumer::TranslateConsumerMessageRelease(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { - Y_UNUSED(subconsumerIndex); - promise.SetValue(std::move(subconsumerResponse)); - return true; -} - -bool TMultiClusterConsumer::TranslateConsumerMessageError(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { - WARN_LOG("Got error from subconsumer " << subconsumerIndex << ": " << subconsumerResponse.Response.GetError(), "", SessionId); - Destroy(subconsumerResponse.Response.GetError()); - Y_UNUSED(promise); - return false; -} - -bool TMultiClusterConsumer::TranslateConsumerMessageStatus(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { - Y_UNUSED(subconsumerIndex); - promise.SetValue(std::move(subconsumerResponse)); - return true; -} - -void TMultiClusterConsumer::RequestSubconsumers() { - const size_t inflight = Requests.size(); - if (!inflight) { - return; - } - TMaybe<std::weak_ptr<TMultiClusterConsumer>> maybeSelf; - for (size_t subconsumerIndex = 0; subconsumerIndex < Subconsumers.size(); ++subconsumerIndex) { - TSubconsumerInfo& consumerInfo = Subconsumers[subconsumerIndex]; - if (!consumerInfo.StartResponseWasGot) { - continue; // Consumer hasn't started yet. - } - while (consumerInfo.PendingRequests.size() < inflight) { - if (!maybeSelf) { - maybeSelf.ConstructInPlace(shared_from_this()); // Don't construct it if we don't need it. - } - consumerInfo.PendingRequests.push_back(consumerInfo.Consumer->GetNextMessage()); - PQLib->Subscribe(consumerInfo.PendingRequests.back(), - this, - [self = *maybeSelf, subconsumerIndex](const auto&) { - if (auto selfShared = self.lock()) { - selfShared->ProcessReadyResponses(subconsumerIndex); - } - }); - } - } -} - -void TMultiClusterConsumer::Destroy(const TError& description) { - if (State == EState::Dead) { - return; - } - State = EState::Dead; - - WARN_LOG("Destroying consumer with error description: " << description, "", SessionId); - - StartPromise.TrySetValue(MakeCreateResponse(description)); - - if (StartDeadlineCallback) { - StartDeadlineCallback->TryCancel(); - } - - for (auto& reqPromise : Requests) { - reqPromise.SetValue(MakeResponse(description)); - } - Requests.clear(); - Subconsumers.clear(); - - IsDeadPromise.SetValue(description); - - DestroyPQLibRef(); -} - -void TMultiClusterConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) { - Destroy(MakeError(description, code)); -} - -void TMultiClusterConsumer::Destroy() { - Destroy(GetCancelReason()); -} - -void TMultiClusterConsumer::Cancel() { - Destroy(GetCancelReason()); -} - -TError TMultiClusterConsumer::MakeError(const TString& description, NErrorCode::EErrorCode code) { - TError error; - error.SetDescription(description); - error.SetCode(code); - return error; -} - -TConsumerCreateResponse TMultiClusterConsumer::MakeCreateResponse(const TError& description) { - TReadResponse res; - res.MutableError()->CopyFrom(description); - return TConsumerCreateResponse(std::move(res)); -} - -TConsumerMessage TMultiClusterConsumer::MakeResponse(const TError& description) { - TReadResponse res; - res.MutableError()->CopyFrom(description); - return TConsumerMessage(std::move(res)); -} - -TMultiClusterConsumer::TSubconsumerInfo::TSubconsumerInfo(std::shared_ptr<IConsumerImpl> consumer) - : Consumer(std::move(consumer)) - , StartFuture(Consumer->Start()) - , DeadFuture(Consumer->IsDead()) -{ -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h deleted file mode 100644 index ab8a338fbe0..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h +++ /dev/null @@ -1,93 +0,0 @@ -#pragma once -#include "channel_p.h" -#include "scheduler.h" -#include "iconsumer_p.h" -#include "internals.h" - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> - -namespace NPersQueue { - -class TPQLibPrivate; - -class TMultiClusterConsumer: public IConsumerImpl, public std::enable_shared_from_this<TMultiClusterConsumer> { -public: - struct TSubconsumerInfo; - struct TCookieMappingItem; - struct TCookieMapping; - -public: - TMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); - ~TMultiClusterConsumer(); - - NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override; - NThreading::TFuture<TError> IsDead() noexcept override; - - void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; - void Commit(const TVector<ui64>& cookies) noexcept override; - void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; - - void Cancel() override; - -private: - void PatchSettings(); - void StartClusterDiscovery(TInstant deadline); - void OnClusterDiscoveryDone(TInstant deadline); - void OnStartTimeout(); - void OnSubconsumerStarted(size_t subconsumerIndex); - - void CheckReadyResponses(); - bool ProcessReadyResponses(size_t subconsumerIndex); // False if consumer has been destroyed. - void RequestSubconsumers(); // Make requests inflight in every subconsumer equal to our superconsumer inflight. - - bool TranslateConsumerMessage(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); // False if consumer has been destroyed. - bool TranslateConsumerMessageLock(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); - bool TranslateConsumerMessageRelease(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); - bool TranslateConsumerMessageData(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); - bool TranslateConsumerMessageCommit(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); - bool TranslateConsumerMessageError(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); // Calls destroy on consumer - bool TranslateConsumerMessageStatus(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); - - void Destroy(const TError& description); - void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); - void Destroy(); - - static TError MakeError(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); - static TConsumerCreateResponse MakeCreateResponse(const TError& description); - static TConsumerMessage MakeResponse(const TError& description); - - void ScheduleClusterDiscoveryRetry(TInstant deadline); - -private: - TConsumerSettings Settings; - TIntrusivePtr<ILogger> Logger; - TString SessionId; - - NThreading::TPromise<TConsumerCreateResponse> StartPromise = NThreading::NewPromise<TConsumerCreateResponse>(); - NThreading::TPromise<TError> IsDeadPromise = NThreading::NewPromise<TError>(); - - enum class EState { - Created, // Before Start() was created. - WaitingClusterDiscovery, // After Start() was called and before we received CDS response. - StartingSubconsumers, // After we received CDS response and before consumer actually started. - Working, // When one or more consumers had started. - Dead, - }; - EState State = EState::Created; - - TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback; - TChannelImplPtr ClusterDiscoverer; - TConsumerChannelOverCdsImpl::TResultPtr ClusterDiscoverResult; - size_t ClusterDiscoveryAttemptsDone = 0; - - std::vector<TSubconsumerInfo> Subconsumers; - size_t CurrentSubconsumer = 0; // Index of current subconsumer to take responses from - - THolder<TCookieMapping> CookieMapping; - THashMap<TString, size_t> OldTopicName2Subconsumer; // Used in lock session for requesting partition status. - - std::deque<NThreading::TPromise<TConsumerMessage>> Requests; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp deleted file mode 100644 index a29a18cbafd..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp +++ /dev/null @@ -1,227 +0,0 @@ -#include "multicluster_consumer.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/event.h> - -namespace NPersQueue { - -Y_UNIT_TEST_SUITE(TMultiClusterConsumerTest) { - TConsumerSettings MakeFakeConsumerSettings() { - TConsumerSettings settings; - settings.Topics.push_back("topic"); - settings.ReadFromAllClusterSources = true; - settings.ReadMirroredPartitions = false; - settings.MaxAttempts = 1; // Consumer should fix this setting. Check it. - return settings; - } - - TConsumerSettings MakeConsumerSettings(const SDKTestSetup& setup) { - TConsumerSettings settings = setup.GetConsumerSettings(); - settings.ReadFromAllClusterSources = true; - settings.ReadMirroredPartitions = false; - settings.MaxAttempts = 1; // Consumer should fix this setting. Check it. - return settings; - } - - Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) { - // Test that consumer doesn't hang on till shutdown - TPQLib lib; - lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); - } - - Y_UNIT_TEST(StartedConsumerCanBeDestructed) { - // Test that consumer doesn't hang on till shutdown - TPQLib lib; - auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); - auto isDead = consumer->IsDead(); - consumer->Start(); - consumer = nullptr; - isDead.GetValueSync(); - } - - Y_UNIT_TEST(CommitRightAfterStart) { - SDKTestSetup setup("InfiniteStart"); - TPQLib lib; - auto settings = MakeConsumerSettings(setup); - settings.Topics[0] = "unknown_topic"; - auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); - consumer->Start(); - auto isDead = consumer->IsDead(); - consumer->Commit({42}); - if (GrpcV1EnabledByDefault()) { - UNIT_ASSERT_EQUAL_C(isDead.GetValueSync().GetCode(), NErrorCode::WRONG_COOKIE, isDead.GetValueSync()); - } else { - UNIT_ASSERT_STRINGS_EQUAL_C(isDead.GetValueSync().GetDescription(), "Requesting commit, but consumer is not in working state", isDead.GetValueSync()); - } - } - - Y_UNIT_TEST(GetNextMessageRightAfterStart) { - if (GrpcV1EnabledByDefault()) { - return; - } - SDKTestSetup setup("InfiniteStart"); - TPQLib lib; - auto settings = MakeConsumerSettings(setup); - settings.Topics[0] = "unknown_topic"; - auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); - consumer->Start(); - auto nextMessage = consumer->GetNextMessage(); - UNIT_ASSERT_STRINGS_EQUAL(nextMessage.GetValueSync().Response.GetError().GetDescription(), "Requesting next message, but consumer is not in working state"); - } - - Y_UNIT_TEST(RequestPartitionStatusRightAfterStart) { - if (GrpcV1EnabledByDefault()) { - return; - } - SDKTestSetup setup("InfiniteStart"); - TPQLib lib; - auto settings = MakeConsumerSettings(setup); - settings.Topics[0] = "unknown_topic"; - auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); - consumer->Start(); - auto isDead = consumer->IsDead(); - consumer->RequestPartitionStatus("topic", 42, 42); - UNIT_ASSERT_STRINGS_EQUAL(isDead.GetValueSync().GetDescription(), "Requesting partition status, but consumer is not in working state"); - } - - Y_UNIT_TEST(StartedConsumerCantBeStartedAgain) { - TPQLib lib; - auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); - auto startFuture = consumer->Start(); - auto startFuture2 = consumer->Start(); - UNIT_ASSERT(startFuture2.HasValue()); - UNIT_ASSERT_STRINGS_EQUAL(startFuture2.GetValueSync().Response.GetError().GetDescription(), "Start was already called"); - } - - Y_UNIT_TEST(StartWithFailedCds) { - TPQLib lib; - auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); - auto startFuture = consumer->Start(TDuration::MilliSeconds(10)); - if (!GrpcV1EnabledByDefault()) { - UNIT_ASSERT_STRING_CONTAINS(startFuture.GetValueSync().Response.GetError().GetDescription(), " cluster discovery attempts"); - } - auto isDead = consumer->IsDead(); - isDead.Wait(); - } - - Y_UNIT_TEST(StartDeadline) { - if (GrpcV1EnabledByDefault()) { - return; - } - TPQLib lib; - TConsumerSettings settings = MakeFakeConsumerSettings(); - auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); - auto startFuture = consumer->Start(TDuration::MilliSeconds(100)); - UNIT_ASSERT_EQUAL_C(startFuture.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, startFuture.GetValueSync().Response); - } - - Y_UNIT_TEST(StartDeadlineAfterCds) { - SDKTestSetup setup("StartDeadlineAfterCds"); - TPQLib lib; - auto settings = MakeConsumerSettings(setup); - settings.Topics[0] = "unknown_topic"; - auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); - auto startFuture = consumer->Start(TDuration::MilliSeconds(1)); - if (!GrpcV1EnabledByDefault()) { - UNIT_ASSERT_STRINGS_EQUAL(startFuture.GetValueSync().Response.GetError().GetDescription(), "Start timeout"); - } - auto isDead = consumer->IsDead(); - isDead.Wait(); - } - - void WorksWithSingleDc(bool oneDcIsUnavailable) { - SDKTestSetup setup("WorksWithSingleDc", false); - if (!oneDcIsUnavailable) { - setup.SetSingleDataCenter(); // By default one Dc is fake in test setup. This call overrides it. - } - setup.Start(); - auto consumer = setup.StartConsumer(MakeConsumerSettings(setup)); - setup.WriteToTopic({"msg1", "msg2"}); - setup.ReadFromTopic({{"msg1", "msg2"}}, true, consumer.Get()); - } - - Y_UNIT_TEST(WorksWithSingleDc) { - WorksWithSingleDc(false); - } - - Y_UNIT_TEST(WorksWithSingleDcWhileSecondDcIsUnavailable) { - WorksWithSingleDc(true); - } - - Y_UNIT_TEST(FailsOnCommittingWrongCookie) { - SDKTestSetup setup("FailsOnCommittingWrongCookie"); - auto consumer = setup.StartConsumer(MakeConsumerSettings(setup)); - setup.WriteToTopic({"msg1", "msg2"}); - auto nextMessage = consumer->GetNextMessage(); - UNIT_ASSERT_EQUAL(nextMessage.GetValueSync().Type, EMT_DATA); - auto isDead = consumer->IsDead(); - consumer->Commit({nextMessage.GetValueSync().Response.GetData().GetCookie() + 1}); - UNIT_ASSERT_STRING_CONTAINS(isDead.GetValueSync().GetDescription(), "Wrong cookie"); - } - - // Test that consumer is properly connected with decompressing consumer. - void Unpacks(bool unpack) { - SDKTestSetup setup(TStringBuilder() << "Unpacks(" << (unpack ? "true)" : "false)")); - setup.WriteToTopic({"msg"}); - auto settings = MakeConsumerSettings(setup); - settings.Unpack = unpack; - auto consumer = setup.StartConsumer(settings); - auto readResponse = consumer->GetNextMessage().GetValueSync(); - UNIT_ASSERT_C(!readResponse.Response.HasError(), readResponse.Response); - UNIT_ASSERT_C(readResponse.Response.HasData(), readResponse.Response); - if (unpack) { - UNIT_ASSERT_EQUAL_C(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), NPersQueueCommon::RAW, readResponse.Response); - UNIT_ASSERT_STRINGS_EQUAL(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg"); - } else { - UNIT_ASSERT_EQUAL_C(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), NPersQueueCommon::GZIP, readResponse.Response); - } - } - - Y_UNIT_TEST(UnpacksIfNeeded) { - Unpacks(true); - } - - Y_UNIT_TEST(DoesNotUnpackIfNeeded) { - Unpacks(false); - } - - Y_UNIT_TEST(WorksWithManyDc) { - SDKTestSetup setup1("WorksWithManyDc1", false); - SDKTestSetup setup2("WorksWithManyDc2"); - SDKTestSetup setup3("WorksWithManyDc3", false); - setup1.AddDataCenter("dc2", setup2); - setup1.AddDataCenter("dc3", setup3); - setup3.GetGrpcServerOptions().SetGRpcShutdownDeadline(TDuration::Seconds(1)); - setup3.Start(); - setup1.Start(); - - // Check that consumer reads from all sources. - setup1.WriteToTopic({"msg1", "msg2"}); - setup2.WriteToTopic({"msg3", "msg4"}); - setup3.WriteToTopic({"msg10", "msg11", "msg12", "msg13", "msg14", "msg15"}); - auto consumer = setup1.StartConsumer(MakeConsumerSettings(setup1)); - setup1.ReadFromTopic({{"msg1", "msg2"}, {"msg3", "msg4"}, {"msg10", "msg11", "msg12", "msg13", "msg14", "msg15"}}, true, consumer.Get()); - - // Turn Dc off and check that consumer works. - setup1.GetLog() << TLOG_INFO << "Turn off DataCenter dc3"; - setup3.ShutdownGRpc(); - setup1.WriteToTopic({"a", "b"}); - setup2.WriteToTopic({"x", "y", "z"}); - setup1.ReadFromTopic({{"a", "b"}, {"x", "y", "z"}}, true, consumer.Get()); - - // Turn Dc on again and check work again. - setup1.GetLog() << TLOG_INFO << "Turn on DataCenter dc3"; - setup3.EnableGRpc(); - setup1.WriteToTopic({"1", "2"}); - setup3.WriteToTopic({"42"}); - setup1.ReadFromTopic({{"1", "2"}, {"42"}}, true, consumer.Get()); - } -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp deleted file mode 100644 index 6f63b0ab4a4..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp +++ /dev/null @@ -1,423 +0,0 @@ -#include "multicluster_producer.h" -#include "persqueue_p.h" - -#include <util/string/builder.h> - -#include <algorithm> - -namespace NPersQueue { - -TWeightIndex::TWeightIndex(size_t count) - : Weights(count) - , Index(count) - , IsEnabled(count) - , EnabledCnt(0) -{ - Y_VERIFY(count > 0); -} - -void TWeightIndex::SetWeight(size_t i, unsigned weight) { - const unsigned oldWeight = Weights[i]; - const int diff = static_cast<int>(weight) - static_cast<int>(oldWeight); - if (diff != 0) { - Weights[i] = weight; - if (IsEnabled[i]) { - UpdateIndex(i, diff); - } - } -} - -void TWeightIndex::Enable(size_t i) { - if (!IsEnabled[i]) { - ++EnabledCnt; - IsEnabled[i] = true; - UpdateIndex(i, static_cast<int>(Weights[i])); - } -} - -void TWeightIndex::Disable(size_t i) { - if (IsEnabled[i]) { - --EnabledCnt; - IsEnabled[i] = false; - UpdateIndex(i, -static_cast<int>(Weights[i])); - } -} - -size_t TWeightIndex::Choose(unsigned randomNumber) const { - Y_VERIFY(randomNumber < WeightsSum()); - Y_VERIFY(!Index.empty()); - const auto choice = std::upper_bound(Index.begin(), Index.end(), randomNumber); - Y_VERIFY(choice != Index.end()); - return choice - Index.begin(); -} - -void TWeightIndex::UpdateIndex(size_t i, int weightDiff) { - for (size_t j = i; j < Index.size(); ++j) { - Index[j] += weightDiff; - } -} - -static TProducerCommitResponse MakeErrorCommitResponse(const TString& description, TProducerSeqNo seqNo, const TData& data) { - TWriteResponse res; - res.MutableError()->SetDescription(description); - res.MutableError()->SetCode(NErrorCode::ERROR); - return TProducerCommitResponse{seqNo, data, std::move(res)}; -} - -TMultiClusterProducer::TMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) - : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) - , Settings(settings) - , DeadPromise(NThreading::NewPromise<TError>()) - , StartPromise(NThreading::NewPromise<TProducerCreateResponse>()) - , State(EState::Created) - , Logger(std::move(logger)) - , WeightIndex(Settings.ServerWeights.size()) - , FallbackWeightIndex(Settings.ServerWeights.size()) - , Subproducers(Settings.ServerWeights.size()) - , FallbackSubproducers(Settings.ServerWeights.size()) -{ - if (Settings.ServerWeights.size() <= 1) { - ythrow yexception() << "MultiCluster producer is working with several servers. You'd better to create retrying producer in this case"; - } - if (Settings.MinimumWorkingDcsCount == 0) { - ythrow yexception() << "MinimumWorkingDcsCount can't be zero. We want that you have at least something working!"; - } - if (Settings.SourceIdPrefix.empty()) { - ythrow yexception() << "SourceIdPrefix must be nonempty"; - } - - bool reconnectWarningPrinted = false; - for (TMultiClusterProducerSettings::TServerWeight& w : Settings.ServerWeights) { - if (!w.Weight) { - ythrow yexception() << "Server weight must be greater than zero"; - } - if (!w.ProducerSettings.ReconnectOnFailure) { - w.ProducerSettings.ReconnectOnFailure = true; - if (!reconnectWarningPrinted) { - reconnectWarningPrinted = true; - WARN_LOG("ReconnectOnFailure setting is off. Turning it on for multicluster producer.", Settings.SourceIdPrefix, ""); - } - ythrow yexception() << "You must enable ReconnectOnFailure setting"; - } - if (w.ProducerSettings.MaxAttempts == std::numeric_limits<unsigned>::max()) { - ythrow yexception() << "MaxAttempts must be a finite number"; - } - } - - for (size_t i = 0; i < Settings.ServerWeights.size(); ++i) { - WeightIndex.SetWeight(i, Settings.ServerWeights[i].Weight); - FallbackWeightIndex.SetWeight(i, Settings.ServerWeights[i].Weight); - } -} - -TMultiClusterProducer::~TMultiClusterProducer() { - Destroy("Destructor called"); -} - -void TMultiClusterProducer::StartSubproducer(TInstant deadline, size_t i, bool fallback) { - std::weak_ptr<TMultiClusterProducer> self(shared_from_this()); - auto& sp = GetSubproducers(fallback)[i]; - sp.Producer = CreateSubproducer(Settings.ServerWeights[i].ProducerSettings, i, fallback); - sp.StartFuture = sp.Producer->Start(deadline); - sp.DeadFuture = sp.Producer->IsDead(); - PQLib->Subscribe(sp.StartFuture, - this, - [self, i, fallback](const auto&) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnProducerStarted(i, fallback); - } - }); - PQLib->Subscribe(sp.DeadFuture, - this, - [self, i, fallback](const auto&) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnProducerDead(i, fallback); - } - }); -} - -void TMultiClusterProducer::OnNeedProducerRestart(size_t i, bool fallback) { - if (State == EState::Dead) { - return; - } - const auto deadline = TInstant::Now() + Settings.ServerWeights[i].ProducerSettings.StartSessionTimeout; - StartSubproducer(deadline, i, fallback); -} - -void TMultiClusterProducer::ScheduleProducerRestart(size_t i, bool fallback) { - std::weak_ptr<TMultiClusterProducer> self(shared_from_this()); - PQLib->GetScheduler().Schedule(Settings.ServerWeights[i].ProducerSettings.ReconnectionDelay, - this, - [self, i, fallback] { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnNeedProducerRestart(i, fallback); - } - }); -} - -void TMultiClusterProducer::StartSubproducers(TInstant deadline, bool fallback) { - for (size_t i = 0; i < Settings.ServerWeights.size(); ++i) { - StartSubproducer(deadline, i, fallback); - } -} - -NThreading::TFuture<TProducerCreateResponse> TMultiClusterProducer::Start(TInstant deadline) noexcept { - Y_VERIFY(State == EState::Created); - State = EState::Starting; - StartSubproducers(deadline); - StartSubproducers(deadline, true); - return StartPromise.GetFuture(); -} - -void TMultiClusterProducer::OnProducerStarted(size_t i, bool fallback) { - if (State == EState::Dead) { - return; - } - TSubproducerInfo& info = GetSubproducers(fallback)[i]; - info.StartAnswered = true; - DEBUG_LOG("Start subproducer response: " << info.StartFuture.GetValue().Response, Settings.SourceIdPrefix, ""); - if (info.StartFuture.GetValue().Response.HasError()) { - ERR_LOG("Failed to start subproducer[" << i << "]: " << info.StartFuture.GetValue().Response.GetError().GetDescription(), Settings.SourceIdPrefix, ""); - if (State == EState::Starting && !fallback) { - size_t startedCount = 0; - for (const auto& p : Subproducers) { - startedCount += p.StartAnswered; - } - if (startedCount == Subproducers.size() && WeightIndex.EnabledCount() < Settings.MinimumWorkingDcsCount) { - Destroy("Not enough subproducers started successfully"); - return; - } - } - - ScheduleProducerRestart(i, fallback); - } else { - GetWeightIndex(fallback).Enable(i); - if (State == EState::Starting && !fallback && WeightIndex.EnabledCount() >= Settings.MinimumWorkingDcsCount) { - State = EState::Working; - TWriteResponse response; - response.MutableInit(); - StartPromise.SetValue(TProducerCreateResponse(std::move(response))); - } - if (fallback) { - ResendLastHopeQueue(i); - } - } -} - -void TMultiClusterProducer::OnProducerDead(size_t i, bool fallback) { - if (State == EState::Dead) { - return; - } - - WARN_LOG("Subproducer[" << i << "] is dead: " << GetSubproducers(fallback)[i].DeadFuture.GetValue().GetDescription(), Settings.SourceIdPrefix, ""); - GetWeightIndex(fallback).Disable(i); - if (!fallback && WeightIndex.EnabledCount() < Settings.MinimumWorkingDcsCount) { - Destroy("Not enough subproducers is online"); - return; - } - ScheduleProducerRestart(i, fallback); -} - -void TMultiClusterProducer::Destroy(const TString& description) { - TError error; - error.SetCode(NErrorCode::ERROR); - error.SetDescription(description); - Destroy(error); -} - -void TMultiClusterProducer::DestroyWrites(const TError& error, TIntrusiveListWithAutoDelete<TWriteInfo, TDelete>& pendingWrites) { - for (TWriteInfo& wi : pendingWrites) { - TWriteResponse response; - response.MutableError()->CopyFrom(error); - wi.ResponsePromise.SetValue(TProducerCommitResponse(0, std::move(wi.Data), std::move(response))); - } - pendingWrites.Clear(); -} - -void TMultiClusterProducer::Destroy(const TError& error) { - if (State == EState::Dead) { - return; - } - - DEBUG_LOG("Destroying multicluster producer. Reason: " << error, Settings.SourceIdPrefix, ""); - const EState prevState = State; - State = EState::Dead; - SubscribeDestroyed(); - if (prevState == EState::Starting) { - TWriteResponse response; - response.MutableError()->CopyFrom(error); - StartPromise.SetValue(TProducerCreateResponse(std::move(response))); - } - for (size_t i = 0; i < Subproducers.size(); ++i) { - Subproducers[i].Producer = nullptr; - FallbackSubproducers[i].Producer = nullptr; - DestroyWrites(error, FallbackSubproducers[i].LastHopePendingWrites); - } - DestroyWrites(error, PendingWrites); - DeadPromise.SetValue(error); - - DestroyPQLibRef(); -} - -void TMultiClusterProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept { - return promise.SetValue(MakeErrorCommitResponse("MultiCluster producer doesn't allow to write with explicitly specified seq no.", seqNo, data)); -} - -void TMultiClusterProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept { - Y_VERIFY(data.IsEncoded()); - - if (State != EState::Working) { - TWriteResponse res; - res.MutableError()->SetDescription(TStringBuilder() << "producer is " << (State == EState::Dead ? "dead" : "not ready") << ". Marker# PQLib01"); - res.MutableError()->SetCode(NErrorCode::ERROR); - promise.SetValue(TProducerCommitResponse{0, std::move(data), std::move(res)}); - return; - } - - TWriteInfo* wi = new TWriteInfo(); - PendingWrites.PushBack(wi); - - const size_t i = WeightIndex.RandomChoose(); - wi->Data = std::move(data); - wi->ResponsePromise = promise; - DelegateWrite(wi, i); -} - -void TMultiClusterProducer::DelegateWrite(TWriteInfo* wi, size_t i, bool fallback) { - if (wi->History.empty() || wi->History.back() != i) { - wi->History.push_back(i); - } - DEBUG_LOG("Delegate" << (fallback ? " fallback" : "") << " write to subproducer[" << i << "]", Settings.SourceIdPrefix, ""); - wi->ResponseFuture = GetSubproducers(fallback)[i].Producer->Write(wi->Data); - wi->WaitingCallback = true; - - std::weak_ptr<TMultiClusterProducer> self(shared_from_this()); - PQLib->Subscribe(wi->ResponseFuture, - this, - [self, wi](const auto&) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnWriteResponse(wi); - } - }); -} - -void TMultiClusterProducer::ResendLastHopeQueue(size_t i) { - TSubproducerInfo& info = FallbackSubproducers[i]; - if (!info.LastHopePendingWrites.Empty()) { - for (TWriteInfo& wi : info.LastHopePendingWrites) { - if (!wi.WaitingCallback) { - DelegateWrite(&wi, i, true); - } - } - } -} - -void TMultiClusterProducer::OnWriteResponse(TWriteInfo* wi) { - if (State == EState::Dead) { - return; - } - - Y_VERIFY(wi->WaitingCallback); - Y_VERIFY(!wi->History.empty()); - - DEBUG_LOG("Write response: " << wi->ResponseFuture.GetValue().Response, Settings.SourceIdPrefix, ""); - wi->WaitingCallback = false; - if (wi->ResponseFuture.GetValue().Response.HasAck()) { - wi->ResponsePromise.SetValue(wi->ResponseFuture.GetValue()); - wi->Unlink(); - delete wi; - } else if (wi->ResponseFuture.GetValue().Response.HasError()) { - // retry write to other dc - if (wi->LastHope) { - const size_t i = wi->History.back(); - if (FallbackWeightIndex.Enabled(i)) { - DelegateWrite(wi, i, true); - } // else we will retry when this producer will become online. - return; - } - TWeightIndex index = FallbackWeightIndex; - for (size_t i : wi->History) { - index.Disable(i); - } - - // choose new fallback dc - size_t i; - if (index.EnabledCount() == 0) { - i = wi->History.back(); - } else { - i = index.RandomChoose(); - } - - // define whether this dc is the last hope - const bool lastHope = index.EnabledCount() <= 1; - if (lastHope) { - wi->LastHope = true; - wi->Unlink(); - FallbackSubproducers[i].LastHopePendingWrites.PushBack(wi); - } - - DEBUG_LOG("Retrying write to subproducer[" << i << "], lastHope: " << lastHope, Settings.SourceIdPrefix, ""); - DelegateWrite(wi, i, true); - } else { // incorrect structure of response - Y_VERIFY(false); - } -} - -NThreading::TFuture<TError> TMultiClusterProducer::IsDead() noexcept { - return DeadPromise.GetFuture(); -} - -TString TMultiClusterProducer::MakeSourceId(const TStringBuf& prefix, const TStringBuf& srcSourceId, - size_t producerIndex, bool fallback) { - TStringBuilder ret; - ret << prefix << "-" << producerIndex; - if (!srcSourceId.empty()) { - ret << "-"sv << srcSourceId; - } - if (fallback) { - ret << "-multicluster-fallback"sv; - } else { - ret << "-multicluster"sv; - } - return std::move(ret); -} - -std::shared_ptr<IProducerImpl> TMultiClusterProducer::CreateSubproducer(const TProducerSettings& srcSettings, - size_t index, bool fallback) { - TProducerSettings settings = srcSettings; - settings.SourceId = MakeSourceId(Settings.SourceIdPrefix, srcSettings.SourceId, index, fallback); - return PQLib->CreateRawRetryingProducer(settings, DestroyEventRef, Logger); -} - -void TMultiClusterProducer::SubscribeDestroyed() { - NThreading::TPromise<void> promise = ProducersDestroyed; - auto handler = [promise](const auto&) mutable { - promise.SetValue(); - }; - std::vector<NThreading::TFuture<void>> futures; - futures.reserve(Subproducers.size() * 2 + 1); - futures.push_back(DestroyedPromise.GetFuture()); - for (size_t i = 0; i < Subproducers.size(); ++i) { - if (Subproducers[i].Producer) { - futures.push_back(Subproducers[i].Producer->Destroyed()); - } - if (FallbackSubproducers[i].Producer) { - futures.push_back(FallbackSubproducers[i].Producer->Destroyed()); - } - } - WaitExceptionOrAll(futures).Subscribe(handler); -} - -NThreading::TFuture<void> TMultiClusterProducer::Destroyed() noexcept { - return ProducersDestroyed.GetFuture(); -} - -void TMultiClusterProducer::Cancel() { - Destroy(GetCancelReason()); -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h deleted file mode 100644 index 16fcfb6b110..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h +++ /dev/null @@ -1,147 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include "internals.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include "channel.h" -#include "iproducer_p.h" - -#include <library/cpp/threading/future/future.h> - -#include <util/generic/intrlist.h> -#include <util/random/random.h> - -#include <deque> - -namespace NPersQueue { - -// Structure to perform choose by weights -class TWeightIndex { -public: - // Creates zero-weighted disabled index - explicit TWeightIndex(size_t count); - - void SetWeight(size_t i, unsigned weight); - - void Enable(size_t i); - void Disable(size_t i); - - // Chooses by random number in interval [0, WeightsSum()) - size_t Choose(unsigned randomNumber) const; - - size_t RandomChoose() const { - return Choose(RandomNumber<unsigned>(WeightsSum())); - } - - size_t EnabledCount() const { - return EnabledCnt; - } - - bool Enabled(size_t i) const { - return i < IsEnabled.size() ? IsEnabled[i] : false; - } - - unsigned WeightsSum() const { - return Index.back(); - } - -private: - void UpdateIndex(size_t i, int weightDiff); - -private: - std::vector<unsigned> Weights; - std::vector<unsigned> Index; // structure for choosing. Index[i] is sum of enabled weights with index <= i. - std::vector<bool> IsEnabled; - size_t EnabledCnt; -}; - -class TMultiClusterProducer: public IProducerImpl, public std::enable_shared_from_this<TMultiClusterProducer> { -public: - TMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); - ~TMultiClusterProducer(); - - NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; - - using IProducerImpl::Write; - - // Forbidden write - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override; - - // Allowed write - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; - - NThreading::TFuture<TError> IsDead() noexcept override; - - NThreading::TFuture<void> Destroyed() noexcept override; - - void Cancel() override; - -private: - struct TWriteInfo: public TIntrusiveListItem<TWriteInfo> { - NThreading::TPromise<TProducerCommitResponse> ResponsePromise; - NThreading::TFuture<TProducerCommitResponse> ResponseFuture; - bool WaitingCallback = false; - std::vector<size_t> History; // Number of DCs with failed or pending writes - TData Data; - bool LastHope = false; // This DC is the last hope: retry till success. - // If this flag is set, this write info is in the local subproducer queue. - }; - - struct TSubproducerInfo { - size_t Number = std::numeric_limits<size_t>::max(); - std::shared_ptr<IProducerImpl> Producer; - NThreading::TFuture<TProducerCreateResponse> StartFuture; - NThreading::TFuture<TError> DeadFuture; - TIntrusiveListWithAutoDelete<TWriteInfo, TDelete> LastHopePendingWrites; // Local queue with writes only to this DC. - bool StartAnswered = false; - }; - -private: - std::shared_ptr<IProducerImpl> CreateSubproducer(const TProducerSettings& settings, size_t index, bool fallback = false); - - std::vector<TSubproducerInfo>& GetSubproducers(bool fallback = false) { - return fallback ? FallbackSubproducers : Subproducers; - } - - TWeightIndex& GetWeightIndex(bool fallback = false) { - return fallback ? FallbackWeightIndex : WeightIndex; - } - - static TString MakeSourceId(const TStringBuf& prefix, const TStringBuf& srcSourceId, - size_t producerIndex, bool fallback); - void StartSubproducers(TInstant deadline, bool fallback = false); - void StartSubproducer(TInstant deadline, size_t i, bool fallback = false); - void OnProducerStarted(size_t i, bool fallback); - void OnProducerDead(size_t i, bool fallback); - void OnWriteResponse(TWriteInfo* wi); - void ScheduleProducerRestart(size_t i, bool fallback); - void OnNeedProducerRestart(size_t i, bool fallback); // scheduler action - void Destroy(const TString& description); - void Destroy(const TError& error); - void DestroyWrites(const TError& error, TIntrusiveListWithAutoDelete<TWriteInfo, TDelete>& pendingWrites); - void DelegateWrite(TWriteInfo* wi, size_t i, bool fallback = false); - void ResendLastHopeQueue(size_t i); - void SubscribeDestroyed(); - -protected: - TMultiClusterProducerSettings Settings; - NThreading::TPromise<TError> DeadPromise; - NThreading::TPromise<TProducerCreateResponse> StartPromise; - enum class EState { - Created, - Starting, - Working, - Dead, - }; - EState State; - TIntrusivePtr<ILogger> Logger; - TWeightIndex WeightIndex; - TWeightIndex FallbackWeightIndex; - std::vector<TSubproducerInfo> Subproducers; - std::vector<TSubproducerInfo> FallbackSubproducers; - TIntrusiveListWithAutoDelete<TWriteInfo, TDelete> PendingWrites; - NThreading::TPromise<void> ProducersDestroyed = NThreading::NewPromise<void>(); -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp deleted file mode 100644 index 6942681e6a5..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp +++ /dev/null @@ -1,360 +0,0 @@ -#include "multicluster_producer.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/event.h> - -#include <atomic> - -namespace NPersQueue { - -Y_UNIT_TEST_SUITE(TWeightIndexTest) { - Y_UNIT_TEST(Enables) { - TWeightIndex wi(3); - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); - - wi.SetWeight(0, 1); - wi.SetWeight(1, 2); - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); - UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 0); - - wi.Disable(1); - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); - UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 0); - - wi.Enable(0); - wi.Enable(0); // no changes - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1); - - wi.Enable(1); - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 2); - UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 3); - - wi.Disable(1); - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1); - - wi.Disable(1); // no changes - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1); - UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1); - } - - Y_UNIT_TEST(Chooses) { - const size_t weightsCount = 3; - TWeightIndex wi(weightsCount); - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); - unsigned weights[weightsCount] = {1, 2, 3}; - for (size_t i = 0; i < weightsCount; ++i) { - wi.SetWeight(i, weights[i]); - wi.Enable(i); - } - UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), weightsCount); - UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 6); - unsigned cnt[weightsCount] = {}; - for (size_t i = 0; i < wi.WeightsSum(); ++i) { - const size_t choice = wi.Choose(i); - UNIT_ASSERT(choice < weightsCount); // it is index - ++cnt[choice]; - } - for (size_t i = 0; i < weightsCount; ++i) { - UNIT_ASSERT_VALUES_EQUAL_C(weights[i], cnt[i], "i: " << i); - } - } -} - -Y_UNIT_TEST_SUITE(TMultiClusterProducerTest) { - Y_UNIT_TEST(NotStartedProducerCanBeDestructed) { - // Test that producer doesn't hang on till shutdown - TPQLib lib; - TMultiClusterProducerSettings settings; - settings.SourceIdPrefix = "prefix"; - settings.ServerWeights.resize(2); - - { - auto& w = settings.ServerWeights[0]; - w.Weight = 100500; - auto& s = w.ProducerSettings; - s.ReconnectOnFailure = true; - s.Server = TServerSetting{"localhost"}; - s.Topic = "topic1"; - s.MaxAttempts = 10; - } - - { - auto& w = settings.ServerWeights[1]; - w.Weight = 1; - auto& s = w.ProducerSettings; - s.ReconnectOnFailure = true; - s.Server = TServerSetting{"localhost"}; - s.Topic = "topic2"; - s.MaxAttempts = 10; - } - lib.CreateMultiClusterProducer(settings, {}, false); - } - - static void InitServer(TTestServer& testServer) { - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2); - - testServer.WaitInit("topic1"); - } - - static void ConnectsToMinOnlineDcs(size_t min) { - return; // Test is ignored. FIX: KIKIMR-7886 - - TTestServer testServer(false); - InitServer(testServer); - - TMultiClusterProducerSettings settings; - settings.SourceIdPrefix = "prefix"; - settings.ServerWeights.resize(2); - settings.MinimumWorkingDcsCount = min; - - { - auto& w = settings.ServerWeights[0]; - w.Weight = 100500; - auto& s = w.ProducerSettings; - s.ReconnectOnFailure = true; - s.Server = TServerSetting{"localhost", testServer.PortManager->GetPort()}; // port with nothing up - s.Topic = "topic1"; - s.ReconnectionDelay = TDuration::MilliSeconds(1); - s.MaxAttempts = 10; - } - - { - auto& w = settings.ServerWeights[1]; - w.Weight = 1; - auto& s = w.ProducerSettings; - s.ReconnectOnFailure = true; - s.Server = TServerSetting{"localhost", testServer.GrpcPort}; - s.Topic = "topic1"; - s.MaxAttempts = 10; - } - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLib lib; - auto producer = lib.CreateMultiClusterProducer(settings, logger, false); - auto start = producer->Start(); - - if (min == 1) { - UNIT_ASSERT_C(!start.GetValueSync().Response.HasError(), start.GetValueSync().Response); - } else { - UNIT_ASSERT_C(start.GetValueSync().Response.HasError(), start.GetValueSync().Response); - } - - DestroyAndWait(producer); - } - - Y_UNIT_TEST(ConnectsToMinOnlineDcs) { - ConnectsToMinOnlineDcs(1); - } - - Y_UNIT_TEST(FailsToConnectToMinOnlineDcs) { - return; // Test is ignored. FIX: KIKIMR-7886 - - ConnectsToMinOnlineDcs(2); - } - - TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { - TConsumerSettings consumerSettings; - consumerSettings.ClientId = "user"; - consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - consumerSettings.Topics.push_back("topic1"); - consumerSettings.CommitsDisabled = true; - return consumerSettings; - } - - void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { - const TProducerCommitResponse& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); - } - - void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); - } - - std::pair<size_t, size_t> ReadNMessagesAndDestroyConsumers(THolder<IConsumer> consumer1, THolder<IConsumer> consumer2, size_t n) { - std::atomic<size_t> readsDone(0), reads1(0), reads2(0); - TSystemEvent ev; - std::pair<std::atomic<size_t>*, IConsumer*> infos[] = { - std::make_pair(&reads1, consumer1.Get()), - std::make_pair(&reads2, consumer2.Get()) - }; - for (size_t i = 0; i < n; ++i) { - for (auto& info : infos) { - auto* reads = info.first; - info.second->GetNextMessage().Subscribe([&, reads](const NThreading::TFuture<TConsumerMessage>& respFuture) { - ++(*reads); - const size_t newReadsDone = ++readsDone; - if (newReadsDone == n) { - ev.Signal(); - } - if (newReadsDone <= n) { - AssertReadValid(respFuture); - } - }); - } - } - ev.Wait(); - consumer1 = nullptr; - consumer2 = nullptr; - return std::pair<size_t, size_t>(reads1, reads2); - } - - Y_UNIT_TEST(WritesInSubproducers) { - return; // Test is ignored. FIX: KIKIMR-7886 - - TTestServer testServer1(false); - InitServer(testServer1); - - TTestServer testServer2(false); - InitServer(testServer2); - - TMultiClusterProducerSettings settings; - settings.SourceIdPrefix = "producer"; - settings.ServerWeights.resize(2); - settings.MinimumWorkingDcsCount = 1; - - { - auto& w = settings.ServerWeights[0]; - w.Weight = 1; - auto& s = w.ProducerSettings; - s.ReconnectOnFailure = true; - s.Server = TServerSetting{"localhost", testServer1.GrpcPort}; - s.Topic = "topic1"; - s.SourceId = "src0"; - s.MaxAttempts = 5; - s.ReconnectionDelay = TDuration::MilliSeconds(30); - } - - { - auto& w = settings.ServerWeights[1]; - w.Weight = 1; - auto& s = w.ProducerSettings; - s.ReconnectOnFailure = true; - s.Server = TServerSetting{"localhost", testServer2.GrpcPort}; - s.Topic = "topic1"; - s.SourceId = "src1"; - s.MaxAttempts = 2; - s.ReconnectionDelay = TDuration::MilliSeconds(30); - } - - TPQLibSettings pqLibSettings; - pqLibSettings.ThreadsCount = 1; - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLib lib(pqLibSettings); - auto producer = lib.CreateMultiClusterProducer(settings, logger, false); - auto startProducerResult = producer->Start(); - UNIT_ASSERT_C(!startProducerResult.GetValueSync().Response.HasError(), startProducerResult.GetValueSync().Response); - - auto consumer1 = lib.CreateConsumer(MakeConsumerSettings(testServer1), logger, false); - UNIT_ASSERT(!consumer1->Start().GetValueSync().Response.HasError()); - - auto consumer2 = lib.CreateConsumer(MakeConsumerSettings(testServer2), logger, false); - UNIT_ASSERT(!consumer2->Start().GetValueSync().Response.HasError()); - - const size_t writes = 100; - for (size_t i = 0; i < writes; ++i) { - auto write = producer->Write(TString(TStringBuilder() << "blob_1_" << i)); - AssertWriteValid(write); - } - - const std::pair<size_t, size_t> msgs = ReadNMessagesAndDestroyConsumers(std::move(consumer1), std::move(consumer2), writes); - UNIT_ASSERT_C(msgs.first > 30, "msgs.first = " << msgs.first); - UNIT_ASSERT_C(msgs.second > 30, "msgs.second = " << msgs.second); - - consumer1 = lib.CreateConsumer(MakeConsumerSettings(testServer1), logger, false); - UNIT_ASSERT(!consumer1->Start().GetValueSync().Response.HasError()); - - testServer2.ShutdownServer(); - //testServer2.CleverServer->ShutdownGRpc(); - - // Assert that all writes go in 1st DC - for (size_t i = 0; i < writes; ++i) { - auto write = producer->Write(TString(TStringBuilder() << "blob_2_" << i)); - auto read = consumer1->GetNextMessage(); - AssertWriteValid(write); - AssertReadValid(read); - } - - // Assert that all futures will be signalled after producer's death - std::vector<NThreading::TFuture<TProducerCommitResponse>> writesFutures(writes); - for (size_t i = 0; i < writes; ++i) { - writesFutures[i] = producer->Write(TString(TStringBuilder() << "blob_3_" << i)); - } - - auto isDead = producer->IsDead(); - producer = nullptr; - for (auto& f : writesFutures) { - f.GetValueSync(); - } - isDead.GetValueSync(); - - DestroyAndWait(consumer1); - DestroyAndWait(consumer2); - } - - Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { - return; // Test is ignored. FIX: KIKIMR-7886 - - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(); - - const size_t partitions = 1; - testServer.AnnoyingClient->InitRoot(); - testServer.AnnoyingClient->InitDCs(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - testServer.AnnoyingClient->InitSourceIds(); - - testServer.WaitInit("topic1"); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLibSettings pqLibSettings; - pqLibSettings.DefaultLogger = logger; - THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); - - TMultiClusterProducerSettings settings; - settings.SourceIdPrefix = "producer"; - settings.ServerWeights.resize(2); - settings.MinimumWorkingDcsCount = 1; - - for (auto& w : settings.ServerWeights) { - w.Weight = 1; - auto& s = w.ProducerSettings; - s.ReconnectOnFailure = true; - s.Server = TServerSetting{"localhost", testServer.GrpcPort}; - s.Topic = "topic1"; - s.SourceId = "src0"; - s.MaxAttempts = 5; - s.ReconnectionDelay = TDuration::MilliSeconds(30); - } - - auto producer = PQLib->CreateMultiClusterProducer(settings, logger, false); - UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); - auto isDead = producer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - auto write1 = producer->Write(1, TString("blob1")); - auto write2 = producer->Write(2, TString("blob2")); - - PQLib = nullptr; - - UNIT_ASSERT(write1.HasValue()); - UNIT_ASSERT(write2.HasValue()); - - auto write3 = producer->Write(3, TString("blob3")); - UNIT_ASSERT(write3.HasValue()); - UNIT_ASSERT(write3.GetValue().Response.HasError()); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp deleted file mode 100644 index cc1f46d8355..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp +++ /dev/null @@ -1,540 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include "internals.h" -#include "persqueue_p.h" -#include "compat_producer.h" -#include "consumer.h" -#include "multicluster_consumer.h" -#include "producer.h" -#include "retrying_consumer.h" -#include "retrying_producer.h" -#include "multicluster_producer.h" -#include "processor.h" -#include "compressing_producer.h" -#include "decompressing_consumer.h" -#include "local_caller.h" -#include "ydb_sdk_consumer.h" - -#include <util/digest/numeric.h> -#include <util/system/thread.h> - -namespace NPersQueue { - - -TString TPQLib::GetUserAgent() const { - return Impl->GetUserAgent(); -} - -void TPQLib::SetUserAgent(const TString& userAgent) { - Impl->SetUserAgent(userAgent); -} - -TPQLib::TPQLib(const TPQLibSettings& settings) - : Impl(new TPQLibPrivate(settings)) -{} - -TPQLib::~TPQLib() { - AtomicSet(Alive, 0); - Impl->CancelObjectsAndWait(); -} - -#define CHECK_PQLIB_ALIVE() Y_VERIFY(AtomicGet(Alive), "Attempt to use PQLib after/during its destruction.") - -THolder<IProducer> TPQLib::CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { - CHECK_PQLIB_ALIVE(); - return MakeHolder<TPublicProducer>(Impl->CreateMultiClusterProducer(settings, deprecated, std::move(logger))); -} - -THolder<IProducer> TPQLib::CreateProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { - CHECK_PQLIB_ALIVE(); - auto producer = settings.ReconnectOnFailure ? - Impl->CreateRetryingProducer(settings, deprecated, std::move(logger)) : - Impl->CreateProducer(settings, deprecated, std::move(logger)); - return MakeHolder<TPublicProducer>(std::move(producer)); -} - -template <class TConsumerOrProducerSettings> -NYdb::NPersQueue::TPersQueueClient& TPQLibPrivate::GetOrCreatePersQueueClient(const TConsumerOrProducerSettings& settings, const TIntrusivePtr<ILogger>& logger) { - with_lock(Lock) { - if (!CompressExecutor) { - CompressExecutor = NYdb::NPersQueue::CreateThreadPoolExecutorAdapter(CompressionPool); - HandlersExecutor = NYdb::NPersQueue::CreateThreadPoolExecutor(1); - } - TWrapperKey key{settings.Server, settings.CredentialsProvider}; - auto it = Wrappers.find(key); - if (Wrappers.end() == it) { - CreateWrapper(key, logger); - it = Wrappers.find(key); - } - return *it->second.PersQueueClient; - } -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateNewConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger) -{ - if (Settings.EnableGRpcV1 && settings.Server.UseLogbrokerCDS != EClusterDiscoveryUsageMode::DontUse && !settings.CommitsDisabled && settings.Unpack) { - auto ret = std::make_shared<TLocalConsumerImplCaller<TYdbSdkCompatibilityConsumer>>(settings, - GetSelfRefsAreDeadEvent(false), - this, - logger, - GetOrCreatePersQueueClient(settings, logger)); - ret->Init(); - AddToDestroySet(ret); - return ret; - } - return nullptr; -} - -THolder<IConsumer> TPQLib::CreateConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { - CHECK_PQLIB_ALIVE(); - return MakeHolder<TPublicConsumer>(Impl->CreateConsumer(settings, deprecated, std::move(logger))); -} - -THolder<IProcessor> TPQLib::CreateProcessor(const TProcessorSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { - CHECK_PQLIB_ALIVE(); - return MakeHolder<TPublicProcessor>(Impl->CreateProcessor(settings, deprecated, std::move(logger))); -} - -void TPQLib::SetLogger(TIntrusivePtr<ILogger> logger) { - CHECK_PQLIB_ALIVE(); - Impl->SetLogger(std::move(logger)); -} - -#undef CHECK_PQLIB_ALIVE - -static void DoExecute(std::shared_ptr<grpc::CompletionQueue> cq, std::shared_ptr<std::atomic<bool>> shuttingDown) { - TThread::SetCurrentThreadName("pqlib_grpc_thr"); - void* tag; - bool ok; - while (cq->Next(&tag, &ok)) { - IQueueEvent* const ev(static_cast<IQueueEvent*>(tag)); - if (shuttingDown->load() || !ev->Execute(ok)) { - ev->DestroyRequest(); - } - } -} - -TPQLibPrivate::TPQLibPrivate(const TPQLibSettings& settings) - : Settings(settings) - , CQ(std::make_shared<grpc::CompletionQueue>()) - , Scheduler(MakeHolder<TScheduler>(this)) - , Logger(settings.DefaultLogger) -{ - Y_VERIFY(Settings.ThreadsCount > 0); - Y_VERIFY(Settings.GRpcThreads > 0); - Y_VERIFY(Settings.CompressionPoolThreads > 0); - CompressionPool = std::make_shared<TThreadPool>(); - CompressionPool->Start(Settings.CompressionPoolThreads); - QueuePool.Start(Settings.ThreadsCount); - GRpcThreads.reserve(Settings.GRpcThreads); - for (size_t i = 0; i < Settings.GRpcThreads; ++i) { - GRpcThreads.emplace_back([cq = CQ, shuttingDown = ShuttingDown](){ DoExecute(cq, shuttingDown); }); - } - CreateSelfRefsAreDeadPtr(); -} - -TPQLibPrivate::~TPQLibPrivate() { - DEBUG_LOG("Destroying PQLib. Destroying scheduler.", "", ""); - Scheduler.Reset(); - - DEBUG_LOG("Destroying PQLib. Set stop flag.", "", ""); - Stop(); - - if (CQ) { - DEBUG_LOG("Destroying PQLib. Shutting down completion queue.", "", ""); - CQ->Shutdown(); - } - - if (!AtomicGet(HasDeprecatedObjects)) { - DEBUG_LOG("Destroying PQLib. Joining threads.", "", ""); - for (std::thread& thread : GRpcThreads) { - thread.join(); - } - } else { - DEBUG_LOG("Destroying PQLib. Detaching threads.", "", ""); - for (std::thread& thread : GRpcThreads) { - thread.detach(); - } - } - - if (YdbDriver) { - YdbDriver->Stop(); - YdbDriver = nullptr; - } - - DEBUG_LOG("Destroying PQLib. Stopping compression pool.", "", ""); - CompressionPool->Stop(); - - DEBUG_LOG("Destroying PQLib. Stopping queue pool.", "", ""); - QueuePool.Stop(); -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - std::shared_ptr<IProducerImpl> res; - auto rawProducerImpl = CreateRawProducerImpl<TProducer>(settings, refsAreDeadPtr, ChooseLogger(logger)); - rawProducerImpl->SetChannel(channelInfo); - res = std::move(rawProducerImpl); - return res; -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - TChannelHolder t = CreateChannel(settings, ChooseLogger(logger)); - std::shared_ptr<IProducerImpl> res; - - auto rawProducerImpl = CreateRawProducerImpl<TProducer>(settings, refsAreDeadPtr, ChooseLogger(logger)); - rawProducerImpl->SetChannel(t); - res = std::move(rawProducerImpl); - return res; -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create producer", settings.SourceId, ""); - std::shared_ptr<IProducerImpl> rawProducer = CreateRawProducer(settings, channelInfo, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); - compressing->Init(); - if (!deprecated) { - AddToDestroySet(compressing); - } - return compressing; -} - -class TLoggerWrapper : public TLogBackend { -public: - TLoggerWrapper(TIntrusivePtr<ILogger> logger) noexcept - : Logger(logger) - {} - - ~TLoggerWrapper() {} - - void WriteData(const TLogRecord& rec) - { - Logger->Log(TString(rec.Data, rec.Len), "", "", (int)rec.Priority); - } - - void ReopenLog() {} - -private: - TIntrusivePtr<ILogger> Logger; -}; - - -void TPQLibPrivate::CreateWrapper(const TWrapperKey& key, TIntrusivePtr<ILogger> logger) { - auto& wrapper = Wrappers[key]; - const auto& settings = key.Settings; - if (!YdbDriver) { - NYdb::TDriverConfig config; - - config.SetEndpoint(TStringBuilder() << settings.Address << ":" << settings.Port) //TODO BAD - .SetDatabase(settings.Database.empty() ? "/Root" : settings.Database) - .SetNetworkThreadsNum(Settings.GRpcThreads) - .SetClientThreadsNum(Settings.ThreadsCount) - .SetGRpcKeepAliveTimeout(TDuration::Seconds(90)) - .SetGRpcKeepAlivePermitWithoutCalls(true) - .SetDiscoveryMode(NYdb::EDiscoveryMode::Async); - if (logger) - config.SetLog(MakeHolder<TLoggerWrapper>(logger)); - if (settings.UseSecureConnection) - config.UseSecureConnection(settings.CaCert); - - YdbDriver = MakeHolder<NYdb::TDriver>(config); - } - - NYdb::NPersQueue::TPersQueueClientSettings pqSettings; - pqSettings.DefaultCompressionExecutor(CompressExecutor) - .DefaultHandlersExecutor(HandlersExecutor) - .ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Auto) - .DiscoveryEndpoint(TStringBuilder() << settings.Address << ":" << settings.Port) - .Database(settings.Database); - if (key.Provider) { - pqSettings.CredentialsProviderFactory(std::shared_ptr<NYdb::ICredentialsProviderFactory>(new TCredentialsProviderFactoryWrapper(key.Provider))); - } - wrapper.PersQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*YdbDriver, pqSettings); -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateNewProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger) -{ - if (Settings.EnableGRpcV1) { - auto ret = std::make_shared<TLocalProducerImplCaller<TYdbSdkCompatibilityProducer>>(settings, - GetOrCreatePersQueueClient(settings, logger), - GetSelfRefsAreDeadEvent(false), - this); - ret->Init(); - AddToDestroySet(ret); - return ret; - } - return nullptr; -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) -{ - auto newP = CreateNewProducer(settings, ChooseLogger(logger)); - if (newP) return newP; - - DEBUG_LOG("Create producer", settings.SourceId, ""); - std::shared_ptr<IProducerImpl> rawProducer = CreateRawProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); - compressing->Init(); - if (!deprecated) { - AddToDestroySet(compressing); - } - return compressing; -} - -template<typename TRawProducerImpl> -std::shared_ptr<TRawProducerImpl> TPQLibPrivate::CreateRawProducerImpl(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create raw producer", settings.SourceId, ""); - const bool addToDestroySet = refsAreDeadPtr != nullptr; - NThreading::TPromise<TProducerCreateResponse> promise = NThreading::NewPromise<TProducerCreateResponse>(); - auto ret = std::make_shared<TLocalProducerImplCaller<TRawProducerImpl>>(settings, CQ, promise, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); - ret->Init(); - if (addToDestroySet) { - AddToDestroySet(ret); - } - return ret; -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - auto t = CreateChannel(settings.Server, settings.CredentialsProvider, ChooseLogger(logger), settings.PreferLocalProxy); - auto res = CreateRawConsumerImpl(settings, std::move(refsAreDeadPtr), ChooseLogger(logger)); - res->SetChannel(t); - return std::move(res); -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - auto res = CreateRawConsumerImpl(settings, refsAreDeadPtr, ChooseLogger(logger)); - res->SetChannel(channelInfo); - return std::move(res); -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create multicluster consumer", "", ""); - const bool addToDestroySet = refsAreDeadPtr != nullptr; - auto consumer = std::make_shared<TLocalConsumerImplCaller<TMultiClusterConsumer>>(settings, std::move(refsAreDeadPtr), this, logger); - consumer->Init(); - if (addToDestroySet) { - AddToDestroySet(consumer); - } - return std::move(consumer); -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) { - const bool addToDestroySet = refsAreDeadPtr != nullptr; - std::shared_ptr<IConsumerImpl> consumer = std::make_shared<TLocalConsumerImplCaller<TDecompressingConsumer>>(std::move(subconsumer), settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); - consumer->Init(); - if (addToDestroySet) { - AddToDestroySet(consumer); - } - return consumer; -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) { - const bool addToDestroySet = refsAreDeadPtr != nullptr; - std::shared_ptr<IConsumerImpl> consumer = std::make_shared<TLocalConsumerImplCaller<TRetryingConsumer>>(settings, std::move(refsAreDeadPtr), this, logger); - consumer->Init(); - if (addToDestroySet) { - AddToDestroySet(consumer); - } - return consumer; -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateConsumer(const TConsumerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) -{ - if (auto newConsumer = CreateNewConsumer(settings, ChooseLogger(logger))) { - return newConsumer; - } - - DEBUG_LOG("Create consumer", "", ""); - // Create raw consumer. - std::shared_ptr<IConsumerImpl> consumer; - if (settings.ReadFromAllClusterSources) { - if (settings.ReadMirroredPartitions) { - ythrow yexception() << "Can't create consumer with both ReadMirroredPartitions and ReadFromAllClusterSources options"; - } - if (settings.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::DontUse) { - ythrow yexception() << "Can't create consumer with ReadFromAllClusterSources option, but without using cluster discovery"; - } - consumer = CreateRawMultiClusterConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - } else if (settings.ReconnectOnFailure) { - consumer = CreateRawRetryingConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - } else { - consumer = CreateRawConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - } - if (settings.Unpack) { - consumer = CreateDecompressingConsumer(std::move(consumer), settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - } - return consumer; -} - -std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create consumer", "", ""); - std::shared_ptr<IConsumerImpl> consumer = CreateRawConsumer(settings, channelInfo, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - if (settings.Unpack) { - return CreateDecompressingConsumer(std::move(consumer), settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - } - return consumer; -} - -std::shared_ptr<TConsumer> TPQLibPrivate::CreateRawConsumerImpl(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create raw consumer", "", ""); - const bool addToDestroySet = refsAreDeadPtr != nullptr; - NThreading::TPromise<TConsumerCreateResponse> promise = NThreading::NewPromise<TConsumerCreateResponse>(); - std::shared_ptr<TConsumer> consumer = std::make_shared<TLocalConsumerImplCaller<TConsumer>>(settings, CQ, promise, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); - consumer->Init(); - if (addToDestroySet) { - AddToDestroySet(consumer); - } - return consumer; -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create raw retrying producer", settings.SourceId, ""); - const bool addToDestroySet = refsAreDeadPtr != nullptr; - auto producer = std::make_shared<TLocalProducerImplCaller<TRetryingProducer>>(settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); - producer->Init(); - if (addToDestroySet) { - AddToDestroySet(producer); - } - return std::move(producer); -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRetryingProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) -{ - auto newP = CreateNewProducer(settings, ChooseLogger(logger)); - if (newP) return newP; - - DEBUG_LOG("Create retrying producer", settings.SourceId, ""); - std::shared_ptr<IProducerImpl> rawProducer = CreateRawRetryingProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); - compressing->Init(); - if (!deprecated) { - AddToDestroySet(compressing); - } - return compressing; -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create raw multicluster producer", settings.SourceIdPrefix, ""); - const bool addToDestroySet = refsAreDeadPtr != nullptr; - auto producer = std::make_shared<TLocalProducerImplCaller<TMultiClusterProducer>>(settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); - producer->Init(); - if (addToDestroySet) { - AddToDestroySet(producer); - } - return std::move(producer); -} - -std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create multicluster producer", settings.SourceIdPrefix, ""); - if (settings.ServerWeights.empty()) { - ythrow yexception() << "Can't create producer without server"; - } - const ECodec codec = settings.ServerWeights[0].ProducerSettings.Codec; - const int quality = settings.ServerWeights[0].ProducerSettings.Quality; - for (const auto& w : settings.ServerWeights) { - if (w.ProducerSettings.Codec != codec) { - ythrow yexception() << "Can't create multicluster producer with different codecs"; - } - } - - std::shared_ptr<IProducerImpl> rawProducer = CreateRawMultiClusterProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); - std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), codec, quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); - compressing->Init(); - if (!deprecated) { - AddToDestroySet(compressing); - } - return compressing; -} - -std::shared_ptr<IProcessorImpl> TPQLibPrivate::CreateProcessor(const TProcessorSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) -{ - DEBUG_LOG("Create processor", "", ""); - auto processor = std::make_shared<TLocalProcessorImplCaller<TProcessor>>(settings, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); - processor->Init(); - if (!deprecated) { - AddToDestroySet(processor); - } - return processor; -} - -TChannelHolder TPQLibPrivate::CreateChannel( - const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TIntrusivePtr<ILogger> logger, - bool preferLocalProxy -) { - DEBUG_LOG("Create channel", "", ""); - TChannelHolder res; - res.ChannelPtr = new TChannel(server, credentialsProvider, this, ChooseLogger(logger), preferLocalProxy); - res.ChannelInfo = res.ChannelPtr->GetChannel(); - res.ChannelPtr->Start(); - return res; -} - - -TChannelHolder TPQLibPrivate::CreateChannel( - const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool preferLocalProxy -) { - DEBUG_LOG("Create channel", "", ""); - TChannelHolder res; - res.ChannelPtr = new TChannel(settings, this, ChooseLogger(logger), preferLocalProxy); - res.ChannelInfo = res.ChannelPtr->GetChannel(); - res.ChannelPtr->Start(); - return res; -} - -void TPQLibPrivate::CancelObjectsAndWait() { - { - auto guard = Guard(Lock); - DEBUG_LOG("Destroying PQLib. Cancelling objects: " << SyncDestroyedSet.size(), "", ""); - for (auto& ptr : SyncDestroyedSet) { - auto ref = ptr.lock(); - if (ref) { - ref->Cancel(); - } - } - SelfRefsAreDeadPtr = nullptr; - } - DEBUG_LOG("Destroying PQLib. Waiting refs to die", "", ""); - SelfRefsAreDeadEvent.Wait(); - - DEBUG_LOG("Destroying PQLib. HasDeprecatedObjects: " << AtomicGet(HasDeprecatedObjects) << ". Refs to PQLibPrivate: " << RefCount(), "", ""); - if (!AtomicGet(HasDeprecatedObjects)) { - Y_ASSERT(RefCount() == 1); - } -} - -void TPQLibPrivate::CreateSelfRefsAreDeadPtr() { - void* fakeAddress = &SelfRefsAreDeadEvent; - SelfRefsAreDeadPtr = std::shared_ptr<void>(fakeAddress, - [event = SelfRefsAreDeadEvent](void*) mutable { - event.Signal(); - }); -} - -void TPQLibPrivate::AddToDestroySet(std::weak_ptr<TSyncDestroyed> obj) { - auto guard = Guard(Lock); - // Check dead objects first - if (DestroySetAddCounter * 2 > SyncDestroyedSet.size()) { - for (auto i = SyncDestroyedSet.begin(); i != SyncDestroyedSet.end();) { - if (i->expired()) { - i = SyncDestroyedSet.erase(i); - } else { - ++i; - } - } - DestroySetAddCounter = 0; - } - ++DestroySetAddCounter; - SyncDestroyedSet.emplace(std::move(obj)); -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h deleted file mode 100644 index 4c7145b0480..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h +++ /dev/null @@ -1,316 +0,0 @@ -#pragma once - -#include "channel.h" -#include "internals.h" -#include "iconsumer_p.h" -#include "iprocessor_p.h" -#include "iproducer_p.h" -#include "scheduler.h" -#include "queue_pool.h" - -#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h> - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h> - -#include <library/cpp/threading/future/future.h> - -#include <util/generic/ptr.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/generic/maybe.h> -#include <util/generic/queue.h> -#include <util/generic/guid.h> -#include <util/generic/hash_set.h> -#include <util/system/types.h> -#include <util/system/mutex.h> -#include <util/thread/factory.h> -#include <util/thread/pool.h> - -#include <util/datetime/base.h> - -#include <grpc++/create_channel.h> -#include <grpc++/completion_queue.h> -#include <google/protobuf/message.h> - -#include <atomic> -#include <type_traits> -#include <memory> -#include <set> -#include <thread> - -namespace NPersQueue { - -class TProducer; -class TConsumer; - - -struct TWrapperKey { - TServerSetting Settings; - std::shared_ptr<ICredentialsProvider> Provider; - - bool operator <(const TWrapperKey& b) const { - return Settings < b.Settings || (!(b.Settings < Settings) && Provider < b.Provider); - } - -}; - - - -class TCredentialsProviderWrapper : public NYdb::ICredentialsProvider { -public: - TCredentialsProviderWrapper(std::shared_ptr<NPersQueue::ICredentialsProvider> provider) - : Provider(provider) - {} - - bool IsValid() const override { - return true; - } - - TString GetAuthInfo() const override { - return GetToken(Provider.get()); - } - - private: - std::shared_ptr<NPersQueue::ICredentialsProvider> Provider; -}; - - -class TCredentialsProviderFactoryWrapper : public NYdb::ICredentialsProviderFactory { -public: - TString GetClientIdentity() const override { - return Guid; - } - - std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { - return Provider; - }; - - - TCredentialsProviderFactoryWrapper(std::shared_ptr<NPersQueue::ICredentialsProvider> provider) - : Provider(new TCredentialsProviderWrapper(provider)) - , Guid(CreateGuidAsString()) - {} - -private: - std::shared_ptr<NYdb::ICredentialsProvider> Provider; - TString Guid; -}; - -class TPQLibPrivate: public TAtomicRefCount<TPQLibPrivate> { - template <class T> - class TBaseFutureSubscriberObjectInQueue: public IObjectInQueue { - public: - void SetFuture(const NThreading::TFuture<T>& future) { - Future = future; - } - - protected: - NThreading::TFuture<T> Future; - }; - - template <class T, class TFunc> - class TFutureSubscriberObjectInQueue: public TBaseFutureSubscriberObjectInQueue<T> { - public: - template <class TF> - TFutureSubscriberObjectInQueue(TF&& f) - : Func(std::forward<TF>(f)) - { - } - - void Process(void*) override { - THolder<IObjectInQueue> holder(this); // like in TOwnedObjectInQueue::Process() - Func(this->Future); - } - - private: - TFunc Func; - }; - - template <class T, class TFunc> - static THolder<TBaseFutureSubscriberObjectInQueue<T>> MakeFutureSubscribeObjectInQueue(TFunc&& f) { - return MakeHolder<TFutureSubscriberObjectInQueue<T, std::remove_reference_t<TFunc>>>(std::forward<TFunc>(f)); - } - - template <class T> - class TSubscriberFunc { - public: - TSubscriberFunc(TPQLibPrivate* pqLib, const void* queueTag, THolder<TBaseFutureSubscriberObjectInQueue<T>> callback) - : PQLib(pqLib) - , QueueTag(queueTag) - , Callback(std::make_shared<THolder<TBaseFutureSubscriberObjectInQueue<T>>>(std::move(callback))) - { - } - - void operator()(const NThreading::TFuture<T>& future) { - if (Callback) { - (*Callback)->SetFuture(future); - THolder<IObjectInQueue> callback(std::move(*Callback)); - const bool added = PQLib->GetQueuePool().GetQueue(QueueTag).Add(callback.Get()); - if (added) { // can be false on shutdown - Y_UNUSED(callback.Release()); - } - } - } - - private: - TPQLibPrivate* PQLib; - const void* QueueTag; - // std::shared_ptr is only because copy constructor is required - std::shared_ptr<THolder<TBaseFutureSubscriberObjectInQueue<T>>> Callback; - }; - -public: - TPQLibPrivate(const TPQLibSettings& settings); - TPQLibPrivate(const TPQLibPrivate&) = delete; - TPQLibPrivate(TPQLibPrivate&&) = delete; - ~TPQLibPrivate(); - - TChannelHolder CreateChannel(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, - TIntrusivePtr<ILogger> logger, bool preferLocalProxy = false); - TChannelHolder CreateChannel(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, - bool preferLocalProxy = false); - - - std::shared_ptr<IProducerImpl> CreateRawProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IProducerImpl> CreateRawProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IProducerImpl> CreateProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IProducerImpl> CreateProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger); - - std::shared_ptr<IProducerImpl> CreateNewProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger); - - std::shared_ptr<IProducerImpl> CreateRawRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IProducerImpl> CreateRetryingProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); - - std::shared_ptr<IProducerImpl> CreateRawMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IProducerImpl> CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); - - std::shared_ptr<IConsumerImpl> CreateRawConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IConsumerImpl> CreateRawConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IConsumerImpl> CreateRawRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IConsumerImpl> CreateRawMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - - std::shared_ptr<IConsumerImpl> CreateConsumer(const TConsumerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IConsumerImpl> CreateConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger); - - std::shared_ptr<IConsumerImpl> CreateNewConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger); - - std::shared_ptr<IProcessorImpl> CreateProcessor(const TProcessorSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); - - TString GetUserAgent() const { - auto guard = Guard(UALock); - return UserAgent; - } - - void SetUserAgent(const TString& userAgent) { - auto guard = Guard(UALock); - UserAgent = userAgent; - } - - void Stop() { - ShuttingDown->store(true); - } - - const std::shared_ptr<grpc::CompletionQueue>& GetCompletionQueue() const { - return CQ; - } - - const TPQLibSettings& GetSettings() const { - return Settings; - } - - TScheduler& GetScheduler() { - return *Scheduler; - } - - TQueuePool& GetQueuePool() { - return QueuePool; - } - - IThreadPool& GetCompressionPool() { - return *CompressionPool; - } - - template <class TFunc, class T> - void Subscribe(const NThreading::TFuture<T>& future, const void* queueTag, TFunc&& func) { - TSubscriberFunc<T> f(this, queueTag, MakeFutureSubscribeObjectInQueue<T>(std::forward<TFunc>(func))); - future.Subscribe(std::move(f)); - } - - void SetLogger(TIntrusivePtr<ILogger> logger) { - Logger = std::move(logger); - } - - TIntrusivePtr<ILogger> ChooseLogger(TIntrusivePtr<ILogger> logger) { - return logger ? std::move(logger) : Logger; - } - - void CancelObjectsAndWait(); - - // for testing - std::shared_ptr<void> GetSelfRefsAreDeadPtr() const { - return SelfRefsAreDeadPtr; - } - - void AddToDestroySet(std::weak_ptr<TSyncDestroyed> obj); - -private: - template<typename TRawProducerImpl> - std::shared_ptr<TRawProducerImpl> CreateRawProducerImpl(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<TConsumer> CreateRawConsumerImpl(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - std::shared_ptr<IConsumerImpl> CreateDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); - - void CreateWrapper(const TWrapperKey& key, TIntrusivePtr<ILogger> logger); - - template <class TConsumerOrProducerSettings> - NYdb::NPersQueue::TPersQueueClient& GetOrCreatePersQueueClient(const TConsumerOrProducerSettings& settings, const TIntrusivePtr<ILogger>& logger); - - std::shared_ptr<void> GetSelfRefsAreDeadEvent(bool isObjectDeprecated) { - if (isObjectDeprecated) { - AtomicSet(HasDeprecatedObjects, 1); - } - return isObjectDeprecated ? nullptr : SelfRefsAreDeadPtr; - } - - void CreateSelfRefsAreDeadPtr(); - - using IThreadRef = TAutoPtr<IThreadFactory::IThread>; - -private: - const TPQLibSettings Settings; - - std::shared_ptr<grpc::CompletionQueue> CQ; - std::vector<std::thread> GRpcThreads; - - std::shared_ptr<std::atomic<bool>> ShuttingDown = std::make_shared<std::atomic<bool>>(false); // shared_ptr is for deprecated case when we have leaked threads - - THolder<TScheduler> Scheduler; - TQueuePool QueuePool; - std::shared_ptr<IThreadPool> CompressionPool; - - TIntrusivePtr<ILogger> Logger; - - TAtomic HasDeprecatedObjects = 0; - - TAutoEvent SelfRefsAreDeadEvent; - std::shared_ptr<void> SelfRefsAreDeadPtr; - TAdaptiveLock Lock, UALock; - size_t DestroySetAddCounter = 0; - std::set<std::weak_ptr<TSyncDestroyed>, std::owner_less<std::weak_ptr<TSyncDestroyed>>> SyncDestroyedSet; - - TString UserAgent = "C++ pqlib v0.3"; - - struct TYdbWrapper { - THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; - }; - - THolder<NYdb::TDriver> YdbDriver; - std::map<TWrapperKey, TYdbWrapper> Wrappers; - NYdb::NPersQueue::IExecutor::TPtr CompressExecutor; - NYdb::NPersQueue::IExecutor::TPtr HandlersExecutor; - std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactoryWrapper; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp deleted file mode 100644 index a695c2a8da7..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include "persqueue_p.h" -#include "producer.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <library/cpp/testing/unittest/registar.h> - -namespace NPersQueue { -} -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp deleted file mode 100644 index 1b3ca57fd83..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp +++ /dev/null @@ -1,358 +0,0 @@ -#include "processor.h" -#include <util/string/builder.h> - -namespace NPersQueue { - -template<typename T> -bool IsReady(const NThreading::TFuture<T>& future) -{ - return future.HasValue() || future.HasException(); -} - -template<typename T> -void Reset(NThreading::TFuture<T>& future) -{ - future = NThreading::TFuture<T>(); -} - -template<typename T> -void TProcessor::SafeSubscribe(NThreading::TFuture<T>& future) -{ - std::weak_ptr<TProcessor> self(shared_from_this()); - PQLib->Subscribe( - future, - this, - [self](const auto&) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->ProcessFutures(); - }}); -} - -TProcessor::TProcessor(const TProcessorSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept - : IProcessorImpl(std::move(destroyEventRef), std::move(pqLib)) - , Settings(settings) - , Logger(std::move(logger)) -{} - -TProcessor::~TProcessor() noexcept -{ - SubscribeDestroyed(); - CleanState(); -} - -void TProcessor::Init() -{ - auto selfShared = shared_from_this(); - auto init = [selfShared] { - selfShared->InitImpl(); - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(this).AddFunc(init)); -} - -void TProcessor::InitImpl() -{ - ScheduleIdleSessionsCleanup(); - RecreateConsumer(); - SafeSubscribe(ConsumerIsStarted); - SafeSubscribe(ConsumerIsDead); -} - -// Must be called under lock -void TProcessor::CleanState() noexcept -{ - ProcessingData.clear(); - Reset(ConsumerIsDead); - Reset(ConsumerRequest); - Consumer.reset(); - Producers.clear(); - Data.clear(); - CurrentOriginDataMemoryUsage = 0; - CurrentProcessedDataMemoryUsage = 0; -} - - -bool TProcessor::NextRequest() noexcept -{ - Y_VERIFY(Consumer); - Y_VERIFY(!ConsumerRequest.Initialized()); - - DEBUG_LOG("Current memory usage: " - << "origin messages " << CurrentOriginDataMemoryUsage << "(" << Settings.MaxOriginDataMemoryUsage << "), " - << "processed messages " << CurrentProcessedDataMemoryUsage << "(" << Settings.MaxProcessedDataMemoryUsage << ")\n", - "", ""); - - if (CurrentOriginDataMemoryUsage < Settings.MaxOriginDataMemoryUsage && CurrentProcessedDataMemoryUsage < Settings.MaxProcessedDataMemoryUsage) { - DEBUG_LOG("Issuing consumer request\n", "", ""); - ConsumerRequest = Consumer->GetNextMessage(); - return true; - } else { - return false; - } -} - -bool TProcessor::ProcessConsumerResponse(TOriginData& result, TVector<NThreading::TFuture<TProcessedData>>& processedDataFutures) noexcept -{ - Y_VERIFY(ConsumerRequest.HasValue()); - auto message = ConsumerRequest.ExtractValueSync(); - Reset(ConsumerRequest); - const auto& type = message.Type; - const auto& resp = message.Response; - if (type == EMT_ERROR) { - // Will be handled later via ConsumerIsDead - ERR_LOG("Got error: " << resp << "\n", "", ""); - } else if (type == EMT_RELEASE) { - Y_VERIFY(resp.HasRelease()); - auto t = resp.GetRelease().GetTopic(); - auto p = resp.GetRelease().GetPartition(); - auto g = resp.GetRelease().GetGeneration(); - DEBUG_LOG("Got release for " << t << ":" << p << ", generation " << g << "\n", "", ""); - } else if (type == EMT_LOCK) { - Y_VERIFY(resp.HasLock()); - auto t = resp.GetLock().GetTopic(); - auto p = resp.GetLock().GetPartition(); - auto g = resp.GetLock().GetGeneration(); - DEBUG_LOG("Got lock for " << t << ":" << p << ", generation " << g << "\n", "", ""); - message.ReadyToRead.SetValue(TLockInfo{}); - } else if (type == EMT_DATA) { - Y_VERIFY(resp.HasData()); - Data.push_back(TDataInfo{resp.GetData().GetCookie()}); - for (auto& batch: resp.GetData().GetMessageBatch()) { - TPartition partition(batch.GetTopic(), batch.GetPartition()); - TVector<TOriginMessage>& originMessages = result.Messages[partition]; - TDeque<TProcessingData>& processingData = ProcessingData[partition]; - Data.back().TotalOriginMessages += batch.MessageSize(); - - for (auto& msg: batch.GetMessage()) { - CurrentOriginDataMemoryUsage += msg.ByteSizeLong(); - NThreading::TPromise<TProcessedData> promise = NThreading::NewPromise<TProcessedData>(); - NThreading::TFuture<TProcessedData> future = promise.GetFuture(); - TOriginMessage originMessage = TOriginMessage{msg, promise}; - processedDataFutures.push_back(future); - originMessages.push_back(originMessage); - processingData.push_back(TProcessingData{future, msg.GetOffset(), msg.ByteSizeLong(), &Data.back()}); - } - } - DEBUG_LOG("Processing data message " << resp.GetData().GetCookie() << "\n", "", ""); - return true; - } else if (type == EMT_COMMIT) { - DEBUG_LOG("Got commit", "", ""); - } else { - WARN_LOG("Unknown message type " << int(type) << ", response " << resp << "\n", "", ""); - } - - return false; -} - -// Must be called under lock -void TProcessor::RecreateConsumer() noexcept -{ - if (DestroyedFlag) { - return; - } - - INFO_LOG("Create consumer\n", "", ""); - CleanState(); - - TConsumerSettings consumerSettings = Settings.ConsumerSettings; - consumerSettings.UseLockSession = true; - - Consumer = PQLib->CreateConsumer(consumerSettings, DestroyEventRef == nullptr, Logger); - ConsumerIsStarted = Consumer->Start(); - ConsumerIsDead = Consumer->IsDead(); -} - -void TProcessor::ScheduleIdleSessionsCleanup() noexcept -{ - if (DestroyedFlag) { - return; - } - - std::weak_ptr<TProcessor> self(shared_from_this()); - PQLib->GetScheduler().Schedule(Settings.SourceIdIdleTimeout, - this, - [self] { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->CloseIdleSessions(); - } - }); -} - -void TProcessor::CloseIdleSessions() noexcept -{ - if (DestroyedFlag) { - return; - } - - TInstant now = TInstant::Now(); - for (auto it = Producers.begin(); it != Producers.end();) { - if (it->second.LastWriteTime + Settings.SourceIdIdleTimeout < now) { - INFO_LOG("Close producer for sourceid=" << it->first.SourceId, "", ""); - it = Producers.erase(it); - } else { - ++it; - } - } - ScheduleIdleSessionsCleanup(); -} - -void TProcessor::ProcessFutures() noexcept -{ - if (DestroyedFlag) { - return; - } - - TMaybe<NThreading::TFuture<TConsumerMessage>> consumerRequestFuture; - TVector<NThreading::TFuture<TProcessedData>> processedDataFutures; - TVector<NThreading::TFuture<TProducerCommitResponse>> producerAckFutures; - - if (IsReady(ConsumerIsDead)) { - ERR_LOG("Consumer died with error" << ConsumerIsDead.ExtractValueSync(), "", ""); - RecreateConsumer(); - SafeSubscribe(ConsumerIsStarted); - SafeSubscribe(ConsumerIsDead); - return; - } - - if (IsReady(ConsumerRequest)) { - TOriginData data; - bool hasData = ProcessConsumerResponse(data, processedDataFutures); - if (hasData) { - Requests.front().SetValue(data); - Requests.pop_front(); - } - } - - TInstant now = TInstant::Now(); - for (auto& partitionData: ProcessingData) { - TDeque<TProcessingData>& processingData = partitionData.second; - - while (!processingData.empty() && IsReady(processingData.front().Processed)) { - Y_VERIFY(processingData.front().Processed.HasValue(), "processing future cannot be filled with exception"); - ui64 originMessageOffset = processingData.front().OriginMessageOffset; - CurrentOriginDataMemoryUsage -= processingData.front().OriginMessageSize; - TProcessedData processedData = processingData.front().Processed.ExtractValueSync(); - TDataInfo* dataInfo = processingData.front().DataInfo; - dataInfo->OriginMessagesCounter++; - dataInfo->TotalProcessedMessages += processedData.Messages.size(); - processingData.pop_front(); - for (auto& processedMessage: processedData.Messages) { - TString newSourceId = processedMessage.SourceIdPrefix + partitionData.first.Topic + ":" + ToString(partitionData.first.PartitionId) + "_" + processedMessage.Topic; - TProducerKey producerKey { processedMessage.Topic, newSourceId }; - IProducer* producer = nullptr; - - auto it = Producers.find(producerKey); - if (it == Producers.end()) { - TProducerSettings producerSettings = Settings.ProducerSettings; - producerSettings.Topic = processedMessage.Topic; - producerSettings.SourceId = newSourceId; - producerSettings.PartitionGroup = processedMessage.Group; - producerSettings.ReconnectOnFailure = true; - Producers[producerKey] = TProducerInfo { PQLib->CreateRetryingProducer(producerSettings, DestroyEventRef == nullptr, Logger), {}, now}; - producer = Producers[producerKey].Producer.get(); - producer->Start(); - } else { - producer = it->second.Producer.get(); - } - - CurrentProcessedDataMemoryUsage += processedMessage.Data.size(); - DEBUG_LOG("Write message with seqNo=" << originMessageOffset + 1 << " to sourceId=" << newSourceId, "", ""); - NThreading::TFuture<TProducerCommitResponse> ack = producer->Write(originMessageOffset + 1, std::move(processedMessage.Data)); - producerAckFutures.push_back(ack); - Producers[producerKey].Queue.push_back(TProduceInfo(ack, dataInfo)); - } - } - } - - for (auto& producer: Producers) { - TProducerInfo& producerInfo = producer.second; - while (!producerInfo.Queue.empty() && IsReady(producerInfo.Queue.front().Ack)) { - auto ack = producerInfo.Queue.front().Ack.ExtractValueSync(); - DEBUG_LOG("Got ack for message with seqNo=" << ack.SeqNo << " from sourceId=" << producer.first.SourceId << " " << ack.Response, "", ""); - producerInfo.LastWriteTime = now; - producerInfo.Queue.front().DataInfo->ProcessedMessagesCounter++; - CurrentProcessedDataMemoryUsage -= ack.Data.GetSourceData().size(); - producerInfo.Queue.pop_front(); - } - } - - TVector<ui64> toCommit; - while (!Data.empty() && Data.front().IsReadyForCommit()) { - toCommit.push_back(Data.front().Cookie); - Data.pop_front(); - } - if (!toCommit.empty()) { - Consumer->Commit(toCommit); - } - - if (IsReady(ConsumerIsStarted) && !ConsumerRequest.Initialized() && !Requests.empty()) { - if (NextRequest()) { - consumerRequestFuture = ConsumerRequest; - } - } - - if (consumerRequestFuture.Defined()) { - SafeSubscribe(*consumerRequestFuture.Get()); - } - - for (auto& future: processedDataFutures) { - SafeSubscribe(future); - } - - for (auto& future: producerAckFutures) { - SafeSubscribe(future); - } -} - -void TProcessor::GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept -{ - Y_VERIFY(!DestroyedFlag); - TMaybe<NThreading::TFuture<TConsumerMessage>> consumerRequestFuture; - Requests.push_back(promise); - if (IsReady(ConsumerIsStarted) && !ConsumerRequest.Initialized()) { - if (NextRequest()) { - consumerRequestFuture = ConsumerRequest; - } - } - - if (consumerRequestFuture.Defined()) { - SafeSubscribe(*consumerRequestFuture.Get()); - } -} - -void TProcessor::SubscribeDestroyed() { - NThreading::TPromise<void> promise = ObjectsDestroyed; - auto handler = [promise](const auto&) mutable { - promise.SetValue(); - }; - std::vector<NThreading::TFuture<void>> futures; - futures.reserve(Producers.size() + 2); - futures.push_back(DestroyedPromise.GetFuture()); - if (Consumer) { - futures.push_back(Consumer->Destroyed()); - } - for (const auto& p : Producers) { - if (p.second.Producer) { - futures.push_back(p.second.Producer->Destroyed()); - } - } - WaitExceptionOrAll(futures).Subscribe(handler); -} - -NThreading::TFuture<void> TProcessor::Destroyed() noexcept { - return ObjectsDestroyed.GetFuture(); -} - -void TProcessor::Cancel() { - DestroyedFlag = true; - - for (auto& req : Requests) { - req.SetValue(TOriginData()); - } - Requests.clear(); - - DestroyPQLibRef(); -} - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h deleted file mode 100644 index a6abb827a0f..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include "persqueue_p.h" - -#include <library/cpp/threading/future/future.h> -#include <util/generic/deque.h> -#include <util/generic/map.h> -#include <util/generic/string.h> -#include <util/generic/vector.h> - -namespace NPersQueue { - -struct TDataInfo { - ui64 Cookie; - ui64 TotalOriginMessages; - ui64 OriginMessagesCounter; - ui64 TotalProcessedMessages; - ui64 ProcessedMessagesCounter; - - TDataInfo(ui64 cookie) - : Cookie(cookie) - , TotalOriginMessages(0) - , OriginMessagesCounter(0) - , TotalProcessedMessages(0) - , ProcessedMessagesCounter(0) - {} - - bool IsReadyForCommit() { - return TotalOriginMessages == OriginMessagesCounter && TotalProcessedMessages == ProcessedMessagesCounter; - } -}; - -struct TProducerKey { - TString Topic; - TString SourceId; - - bool operator <(const TProducerKey& other) const { - return std::tie(Topic, SourceId) < std::tie(other.Topic, other.SourceId); - } -}; - -struct TProduceInfo { - NThreading::TFuture<TProducerCommitResponse> Ack; - TDataInfo* DataInfo = nullptr; - - explicit TProduceInfo(const NThreading::TFuture<TProducerCommitResponse>& ack, TDataInfo* dataInfo) - : Ack(ack) - , DataInfo(dataInfo) - {} -}; - -struct TProducerInfo { - std::shared_ptr<IProducerImpl> Producer; - TDeque<TProduceInfo> Queue; - TInstant LastWriteTime; -}; - -struct TProcessingData { - NThreading::TFuture<TProcessedData> Processed; - ui64 OriginMessageOffset; - ui64 OriginMessageSize; - TDataInfo* DataInfo; - - TProcessingData(const NThreading::TFuture<TProcessedData>& processed, ui64 originMessageOffset, ui64 originMessageSize, TDataInfo* dataInfo) - : Processed(processed) - , OriginMessageOffset(originMessageOffset) - , OriginMessageSize(originMessageSize) - , DataInfo(dataInfo) - {}; -}; - -class TPQLibPrivate; - -class TProcessor: public IProcessorImpl, public std::enable_shared_from_this<TProcessor> { -public: - TProcessor(const TProcessorSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept; - ~TProcessor() noexcept; - void Init() override; - - using IProcessorImpl::GetNextData; - void GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept override; - - NThreading::TFuture<void> Destroyed() noexcept override; - - void Cancel() override; - -protected: - TProcessorSettings Settings; - TIntrusivePtr<ILogger> Logger; - std::shared_ptr<IConsumerImpl> Consumer; - NThreading::TFuture<TConsumerCreateResponse> ConsumerIsStarted; - NThreading::TFuture<TError> ConsumerIsDead; - NThreading::TFuture<TConsumerMessage> ConsumerRequest; - TMap<TProducerKey, TProducerInfo> Producers; - ui64 CurrentOriginDataMemoryUsage; - ui64 CurrentProcessedDataMemoryUsage; - - TDeque<NThreading::TPromise<TOriginData>> Requests; - TDeque<TDataInfo> Data; - TMap<TPartition, TDeque<TProcessingData>> ProcessingData; - NThreading::TPromise<void> ObjectsDestroyed = NThreading::NewPromise<void>(); - bool DestroyedFlag = false; - - template<typename T> - void SafeSubscribe(NThreading::TFuture<T>& future); - void CleanState() noexcept; - bool NextRequest() noexcept; - void RecreateConsumer() noexcept; - void ScheduleIdleSessionsCleanup() noexcept; - void CloseIdleSessions() noexcept; - void ProcessFutures() noexcept; - bool ProcessConsumerResponse(TOriginData& result, TVector<NThreading::TFuture<TProcessedData>>& processedDataFutures) noexcept; - void SubscribeDestroyed(); - void InitImpl(); -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp deleted file mode 100644 index d36573dcfc4..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> - -#include <library/cpp/testing/unittest/registar.h> - -using namespace NThreading; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; - -namespace NPersQueue { -Y_UNIT_TEST_SUITE(TProcessorTest) { - TConsumerSettings MakeConsumerSettings(const TString& clientId, const TVector<TString>& topics, - const TTestServer& testServer) { - TConsumerSettings consumerSettings; - consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - consumerSettings.ClientId = clientId; - consumerSettings.Topics = topics; - consumerSettings.UseLockSession = false; - return consumerSettings; - } - - TProducerSettings MakeProducerSettings(const TString& topic, const TTestServer& testServer) { - TProducerSettings producerSettings; - producerSettings.ReconnectOnFailure = true; - producerSettings.Topic = topic; - producerSettings.SourceId = "123"; - producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - producerSettings.Codec = ECodec::RAW; - return producerSettings; - } - - TProcessorSettings MakeProcessorSettings(const TTestServer& testServer) { - TProcessorSettings processorSettings; - processorSettings.ConsumerSettings = MakeConsumerSettings("processor", {"input"}, testServer); - processorSettings.ProducerSettings = MakeProducerSettings("output", testServer); - return processorSettings; - } - - void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { - const TProducerCommitResponse& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); - } - - void ProcessMessage(TOriginData& originData) { - for (auto& messages: originData.Messages) { - for (auto& message: messages.second) { - auto msg = message.Message; - - TVector<TProcessedMessage> processedMessages; - for (int i = 0; i < 3; ++i) { - TStringBuilder sourceIdPrefix; - sourceIdPrefix << "shard" << i << "_"; - processedMessages.push_back(TProcessedMessage {"output", ToString(msg.GetOffset()), sourceIdPrefix, 0}); - } - message.Processed.SetValue(TProcessedData{processedMessages}); - } - } - } - - bool IsFinished(const TTestServer& testServer) { - auto clientInfo = testServer.AnnoyingClient->GetClientInfo({"rt3.dc1--input"}, "processor", true); - auto topicResult = clientInfo.GetMetaResponse().GetCmdGetReadSessionsInfoResult().GetTopicResult(0).GetPartitionResult(); - for (auto& partition: topicResult) { - if (partition.GetClientOffset() < partition.GetEndOffset()) return false; - } - return true; - } - - void BaseFunctionalityTest() { - TTestServer testServer(false); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--input", 2); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--output", 4); - - auto producerSettings = MakeProducerSettings("input", testServer); - auto producer = testServer.PQLib->CreateProducer(producerSettings, testServer.PQLibSettings.DefaultLogger, false); - producer->Start(); - - for (int i = 1; i < 4; ++i) { - auto write1 = producer->Write(i, TString("blob")); - AssertWriteValid(write1); - } - - auto processorSettings = MakeProcessorSettings(testServer); - auto processor = testServer.PQLib->CreateProcessor(processorSettings, testServer.PQLibSettings.DefaultLogger, false); - - while (!IsFinished(testServer)) { - auto data = processor->GetNextData(); - data.Subscribe([](const auto& f) { - auto value = f.GetValueSync(); - ProcessMessage(value); - }); - Sleep(TDuration::MilliSeconds(100)); - } - - auto consumerSettings = MakeConsumerSettings("checker", {"output"}, testServer); - auto consumer = testServer.PQLib->CreateConsumer(consumerSettings, nullptr, false); - auto isStarted = consumer->Start().ExtractValueSync(); - UNIT_ASSERT_C(isStarted.Response.HasInit(), "cannot start consumer " << isStarted.Response); - - ui32 totalMessages = 9; - ui32 currentMessages = 0; - THashMap<TString, ui32> expectedSeqno = { - {"shard0_rt3.dc1--input:0_output", 1}, {"shard1_rt3.dc1--input:0_output", 1}, {"shard2_rt3.dc1--input:0_output", 1}, - {"shard0_rt3.dc1--input:1_output", 1}, {"shard1_rt3.dc1--input:1_output", 1}, {"shard2_rt3.dc1--input:1_output", 1}, - {"shard0_rt3.dc1--input:2_output", 1}, {"shard1_rt3.dc1--input:2_output", 1}, {"shard2_rt3.dc1--input:2_output", 1}, - {"shard0_rt3.dc1--input:3_output", 1}, {"shard1_rt3.dc1--input:3_output", 1}, {"shard2_rt3.dc1--input:3_output", 1}, - }; - while (currentMessages < totalMessages) { - auto f = consumer->GetNextMessage(); - for (auto& batch: f.GetValueSync().Response.GetData().GetMessageBatch()) { - for (auto& message: batch.GetMessage()) { - TString sourceId = message.GetMeta().GetSourceId(); - ui32 seqNo = message.GetMeta().GetSeqNo(); - UNIT_ASSERT_EQUAL(ToString(seqNo - 1), message.GetData()); - auto it = expectedSeqno.find(sourceId); - UNIT_ASSERT_UNEQUAL_C(it, expectedSeqno.end(), "unknown sourceId " << sourceId); - UNIT_ASSERT_EQUAL(it->second, seqNo); - it->second++; - currentMessages += 1; - } - } - } - } - - Y_UNIT_TEST(BasicFunctionality) { - BaseFunctionalityTest(); - } -} -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp deleted file mode 100644 index 2de983453f1..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp +++ /dev/null @@ -1,506 +0,0 @@ -#include <util/generic/strbuf.h> -#include <util/stream/zlib.h> -#include <util/string/cast.h> -#include <util/string/printf.h> -#include <util/string/vector.h> -#include <util/string/builder.h> - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h> - -#include <grpc++/create_channel.h> - - -namespace NPersQueue { - -class TProducerDestroyHandler : public IHandler { -public: - TProducerDestroyHandler(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : Ptr(std::move(ptr)) - , RpcStuff(std::move(rpcStuff)) - , QueueTag(queueTag) - , PQLib(pqLib) - {} - - void Destroy(const TError&) override { - auto producer = Ptr; - auto handler = [producer] { - auto producerShared = producer.lock(); - if (producerShared) { - producerShared->OnFail(); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - - TString ToString() override { return "TProducerDestroyHandler"; } - -protected: - std::weak_ptr<TProducer> Ptr; - TIntrusivePtr<TProducer::TRpcStuff> RpcStuff; // must simply live - const void* QueueTag; - TPQLibPrivate* PQLib; -}; - -class TStreamCreated : public TProducerDestroyHandler { -public: - TStreamCreated(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) - {} - - void Done() override { - auto producer = Ptr; - auto UA = PQLib->GetUserAgent(); - auto handler = [producer, UA] { - auto producerShared = producer.lock(); - if (producerShared) { - producerShared->OnStreamCreated(UA); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - - TString ToString() override { return "StreamCreated"; } -}; - -class TFinishDone : public TProducerDestroyHandler { -public: - TFinishDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) - {} - - void Done() override { - auto producer = Ptr; - auto handler = [producer] { - auto producerShared = producer.lock(); - if (producerShared) { - producerShared->OnFinishDone(); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - - void Destroy(const TError&) override { - Y_FAIL("Finish call failed"); - } - - TString ToString() override { return "Finish"; } -}; - - -class TWriteDone : public TProducerDestroyHandler { -public: - TWriteDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) - {} - - void Done() override { - auto producer = Ptr; - auto handler = [producer] { - auto producerShared = producer.lock(); - if (producerShared) { - producerShared->OnWriteDone(); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - TString ToString() override { return "WriteDone"; } -}; - - -class TReadDone : public TProducerDestroyHandler { -public: - TReadDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, - const void* queueTag, TPQLibPrivate* pqLib) - : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) - {} - - void Done() override { - auto producer = Ptr; - auto handler = [producer] { - auto producerShared = producer.lock(); - if (producerShared) { - producerShared->OnReadDone(); - } - }; - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); - } - TString ToString() override { return "ReadDone"; } -}; - -TProducer::TProducer(const TProducerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, - NThreading::TPromise<TProducerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept - : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) - , RpcStuff(new TRpcStuff()) - , Settings(settings) - , StartPromise(promise) - , IsDeadPromise(NThreading::NewPromise<TError>()) - , Pos(0) - , WriteInflight(false) - , ProxyCookie(0) - , Logger(std::move(logger)) - , IsDestroyed(false) - , IsDestroying(false) - , Failing(false) -{ - RpcStuff->CQ = std::move(cq); -} - -TProducer::~TProducer() { - Destroy(); -} - -void TProducer::Destroy() noexcept { - IsDestroying = true; - RpcStuff->Context.TryCancel(); - ChannelHolder.ChannelPtr = nullptr; - - Destroy("Destructor called"); -} - -void TProducer::DoStart(TInstant deadline) { - if (IsDestroyed) { - return; - } - - if (deadline != TInstant::Max()) { - std::weak_ptr<TProducer> self = shared_from_this(); - auto onStartTimeout = [self] { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnStartTimeout(); - } - }; - StartDeadlineCallback = - PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout); - } - - FillMetaHeaders(RpcStuff->Context, Settings.Server.Database, Settings.CredentialsProvider.get()); - RpcStuff->Stub = PersQueueService::NewStub(RpcStuff->Channel); - RpcStuff->Stream = RpcStuff->Stub->AsyncWriteSession(&RpcStuff->Context, RpcStuff->CQ.get(), new TQueueEvent(StreamCreatedHandler)); -} - -void TProducer::Init() { - std::weak_ptr<TProducer> self = shared_from_this(); // we can't make this object in constructor, because this will be the only reference to us and ~shared_ptr() will destroy us. - StreamCreatedHandler.Reset(new TStreamCreated(self, RpcStuff, this, PQLib.Get())); - ReadDoneHandler.Reset(new TReadDone(self, RpcStuff, this, PQLib.Get())); - WriteDoneHandler.Reset(new TWriteDone(self, RpcStuff, this, PQLib.Get())); - FinishDoneHandler.Reset(new TFinishDone(self, RpcStuff, this, PQLib.Get())); -} - -NThreading::TFuture<TProducerCreateResponse> TProducer::Start(TInstant deadline) noexcept { - if (IsDestroyed) { - TWriteResponse res; - res.MutableError()->SetDescription("Producer is dead"); - res.MutableError()->SetCode(NErrorCode::ERROR); - return NThreading::MakeFuture<TProducerCreateResponse>(TProducerCreateResponse{std::move(res)}); - } - NThreading::TFuture<TProducerCreateResponse> ret = StartPromise.GetFuture(); - if (ChannelHolder.ChannelInfo.Initialized()) { - std::weak_ptr<TProducer> self = shared_from_this(); - PQLib->Subscribe(ChannelHolder.ChannelInfo, - this, - [self, deadline](const auto& f) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->SetChannel(f.GetValue()); - selfShared->DoStart(deadline); - } - }); - } else { - if (RpcStuff->Channel && ProxyCookie) { - DoStart(deadline); - } else { - Destroy("No channel"); - } - } - return ret; -} - -void TProducer::SetChannel(const TChannelHolder& channel) noexcept { - if (IsDestroyed) { - return; - } - - ChannelHolder = channel; -} - -void TProducer::SetChannel(const TChannelInfo& channel) noexcept { - if (IsDestroyed) { - return; - } - - Y_VERIFY(!RpcStuff->Channel); - if (!channel.Channel || channel.ProxyCookie == 0) { - if (channel.Channel) { - Destroy("ProxyCookie is zero"); - } else { - Destroy("Channel creation error"); - } - - return; - } - - RpcStuff->Channel = channel.Channel; - ProxyCookie = channel.ProxyCookie; -} - - -static TProducerCommitResponse MakeErrorCommitResponse(const TString& description, TProducerSeqNo seqNo, const TData& data) { - TWriteResponse res; - res.MutableError()->SetDescription(description); - res.MutableError()->SetCode(NErrorCode::ERROR); - return TProducerCommitResponse{seqNo, data, std::move(res)}; -} - -void TProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept { - return TProducer::Write(promise, MaxSeqNo + 1, std::move(data)); -} - -void TProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept { - Y_VERIFY(data.IsEncoded()); - - MaxSeqNo = Max(seqNo, MaxSeqNo); - - if (IsDestroyed) { - promise.SetValue(MakeErrorCommitResponse("producer is dead", seqNo, data)); - return; - } - - if (StartPromise.Initialized()) { - promise.SetValue(MakeErrorCommitResponse("producer is not ready", seqNo, data)); - return; - } - - CommitPromises.push_back(promise); - Data.emplace_back(seqNo, std::move(data)); - DoWrite(); -} - -void TProducer::OnWriteDone() { - if (IsDestroyed) { - return; - } - - WriteInflight = false; - DoWrite(); -} - -void TProducer::DoWrite() { - if (Failing && !WriteInflight) { - return; - } - if (WriteInflight || Pos == Data.size()) { - return; - } - WriteInflight = true; - TWriteRequest req; - Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials()); - - req.MutableData()->SetSeqNo(Data[Pos].SeqNo); - req.MutableData()->SetData(Data[Pos].Data.GetEncodedData()); - req.MutableData()->SetCodec(Data[Pos].Data.GetCodecType()); - req.MutableData()->SetCreateTimeMs(Data[Pos].Data.GetTimestamp().MilliSeconds()); - - ++Pos; - RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); -} - -NThreading::TFuture<TError> TProducer::IsDead() noexcept { - return IsDeadPromise.GetFuture(); -} - -void TProducer::Destroy(const TString& description) { - TError error; - error.SetDescription(description); - error.SetCode(NErrorCode::ERROR); - Destroy(error); -} - -void TProducer::Destroy(const TError& error) { - if (IsDestroyed) { - return; - } - - if (!IsDestroying) { - INFO_LOG("Error: " << error, Settings.SourceId, SessionId); - } - - IsDestroyed = true; - IsDeadPromise.SetValue(error); - - if (StartDeadlineCallback) { - StartDeadlineCallback->TryCancel(); - } - - NThreading::TFuture<TChannelInfo> tmp; - tmp.Swap(ChannelHolder.ChannelInfo); - ChannelHolder.ChannelPtr = nullptr; - - Error = error; - if (StartPromise.Initialized()) { - NThreading::TPromise<TProducerCreateResponse> tmp; - tmp.Swap(StartPromise); - TWriteResponse res; - res.MutableError()->CopyFrom(Error); - tmp.SetValue(TProducerCreateResponse{std::move(res)}); - } - while (!CommitPromises.empty()) { - auto p = CommitPromises.front(); - CommitPromises.pop_front(); - auto pp(std::move(Data.front())); - Data.pop_front(); - if (Pos > 0) { - --Pos; - } - TWriteResponse res; - res.MutableError()->CopyFrom(Error); - - p.SetValue(TProducerCommitResponse{pp.SeqNo, std::move(pp.Data), std::move(res)}); - } - - StreamCreatedHandler.Reset(); - ReadDoneHandler.Reset(); - WriteDoneHandler.Reset(); - FinishDoneHandler.Reset(); - - DestroyPQLibRef(); -} - -void TProducer::OnStreamCreated(const TString& userAgent) { - if (IsDestroyed) { - return; - } - - TWriteRequest req; - Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials()); - - req.MutableInit()->SetSourceId(Settings.SourceId); - req.MutableInit()->SetTopic(Settings.Topic); - req.MutableInit()->SetProxyCookie(ProxyCookie); - req.MutableInit()->SetPartitionGroup(Settings.PartitionGroup); - req.MutableInit()->SetVersion(userAgent); - Y_VERIFY(!userAgent.empty()); - - for (const auto& attr : Settings.ExtraAttrs) { - auto item = req.MutableInit()->MutableExtraFields()->AddItems(); - item->SetKey(attr.first); - item->SetValue(attr.second); - } - - WriteInflight = true; - RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); - - RpcStuff->Response.Clear(); - RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); -} - -void TProducer::OnFail() { - if (Failing) - return; - Failing = true; - - if (IsDestroyed) { - return; - } - - RpcStuff->Stream->Finish(&RpcStuff->Status, new TQueueEvent(FinishDoneHandler)); -} - -void TProducer::OnFinishDone() { - if (IsDestroyed) { - return; - } - - TError error; - const auto& msg = RpcStuff->Status.error_message(); - TString reason(msg.data(), msg.length()); - error.SetDescription(reason); - error.SetCode(NErrorCode::ERROR); - - Destroy(error); -} - -void TProducer::OnReadDone() { - if (IsDestroyed) { - return; - } - - if (RpcStuff->Response.HasError()) { - Destroy(RpcStuff->Response.GetError()); - return; - } - - if (StartPromise.Initialized()) { //init response - NThreading::TPromise<TProducerCreateResponse> tmp; - tmp.Swap(StartPromise); - Y_VERIFY(RpcStuff->Response.HasInit()); - auto res(std::move(RpcStuff->Response)); - RpcStuff->Response.Clear(); - - SessionId = res.GetInit().GetSessionId(); - const ui32 partition = res.GetInit().GetPartition(); - MaxSeqNo = res.GetInit().GetMaxSeqNo(); - const TProducerSeqNo seqNo = MaxSeqNo; - const TString topic = res.GetInit().GetTopic(); - tmp.SetValue(TProducerCreateResponse{std::move(res)}); - - if (StartDeadlineCallback) { - StartDeadlineCallback->TryCancel(); - } - StartDeadlineCallback = nullptr; - - RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); - - DEBUG_LOG("Stream created to topic " << topic << " partition " << partition << " maxSeqNo " << seqNo, Settings.SourceId, SessionId); - } else { //write response - //get first CommitPromise - Y_VERIFY(!Data.empty()); - auto p = CommitPromises.front(); - CommitPromises.pop_front(); - auto pp(std::move(Data.front())); - Data.pop_front(); - Y_VERIFY(Pos > 0); - --Pos; - - Y_VERIFY(RpcStuff->Response.HasAck()); - Y_VERIFY(RpcStuff->Response.GetAck().GetSeqNo() == pp.SeqNo); - auto res(std::move(RpcStuff->Response)); - RpcStuff->Response.Clear(); - RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); - - p.SetValue(TProducerCommitResponse{pp.SeqNo, std::move(pp.Data), std::move(res)}); - } -} - -void TProducer::OnStartTimeout() { - if (IsDestroyed) { - return; - } - - StartDeadlineCallback = nullptr; - if (!StartPromise.Initialized()) { - // everything is OK, there is no timeout, we have already started. - return; - } - - TError error; - error.SetDescription("Start timeout"); - error.SetCode(NErrorCode::CREATE_TIMEOUT); - Destroy(error); -} - -void TProducer::Cancel() { - IsDestroying = true; - RpcStuff->Context.TryCancel(); - ChannelHolder.ChannelPtr = nullptr; - - Destroy(GetCancelReason()); -} - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h deleted file mode 100644 index af74ced4b9b..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h +++ /dev/null @@ -1,107 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h> -#include <library/cpp/threading/future/future.h> -#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> -#include <deque> - -namespace NPersQueue { - -class TProducer : public IProducerImpl, public std::enable_shared_from_this<TProducer> { -public: - using IProducerImpl::Write; - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept override; - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; - - NThreading::TFuture<TError> IsDead() noexcept override; - - NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; - - TProducer(const TProducerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, - NThreading::TPromise<TProducerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept; - ~TProducer(); - - void Destroy() noexcept; - void SetChannel(const TChannelHolder& channel) noexcept; - void SetChannel(const TChannelInfo& channel) noexcept; - - void DoStart(TInstant deadline); - - void Init() override; - - const TProducerSettings& GetSettings() const { - return Settings; - } - - void Cancel() override; - -public: - using TStream = grpc::ClientAsyncReaderWriterInterface<TWriteRequest, TWriteResponse>; - - // objects that must live after destruction of producer untill all the callbacks arrive at CompletionQueue - struct TRpcStuff: public TAtomicRefCount<TRpcStuff> { - TWriteResponse Response; - std::shared_ptr<grpc::CompletionQueue> CQ; - std::shared_ptr<grpc::Channel> Channel; - std::unique_ptr<PersQueueService::Stub> Stub; - grpc::ClientContext Context; - std::unique_ptr<TStream> Stream; - grpc::Status Status; - }; - -private: - friend class TPQLibPrivate; - friend class TStreamCreated; - friend class TReadDone; - friend class TWriteDone; - friend class TProducerDestroyHandler; - friend class TFinishDone; - - IHandlerPtr StreamCreatedHandler; - IHandlerPtr ReadDoneHandler; - IHandlerPtr WriteDoneHandler; - IHandlerPtr FinishDoneHandler; - - void Destroy(const TError& error); - void Destroy(const TString& description); // the same but with Code=ERROR - void OnStreamCreated(const TString& userAgent); - void OnReadDone(); - void OnWriteDone(); - void DoWrite(); - void OnFail(); - void OnFinishDone(); - void OnStartTimeout(); - -protected: - TIntrusivePtr<TRpcStuff> RpcStuff; - - TChannelHolder ChannelHolder; - TProducerSettings Settings; - - TString SessionId; - - NThreading::TPromise<TProducerCreateResponse> StartPromise; - NThreading::TPromise<TError> IsDeadPromise; - std::deque<NThreading::TPromise<TProducerCommitResponse>> CommitPromises; - std::deque<TWriteData> Data; - ui32 Pos = 0; - bool WriteInflight; - ui64 ProxyCookie = 0; - TProducerSeqNo MaxSeqNo = 0; - - TIntrusivePtr<ILogger> Logger; - - TError Error; - - bool IsDestroyed; - bool IsDestroying; - bool Failing; - TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp deleted file mode 100644 index 5f3deb880d1..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp +++ /dev/null @@ -1,202 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> -#include <library/cpp/testing/unittest/registar.h> - -namespace NPersQueue { - -Y_UNIT_TEST_SUITE(TProducerTest) { - Y_UNIT_TEST(NotStartedProducerCanBeDestructed) { - // Test that producer doesn't hang on till shutdown - TPQLib lib; - TProducerSettings settings; - settings.Server = TServerSetting{"localhost"}; - settings.Topic = "topic"; - settings.SourceId = "src"; - lib.CreateProducer(settings, {}, false); - } - - TProducerSettings MakeProducerSettings(const TTestServer& testServer) { - TProducerSettings producerSettings; - producerSettings.ReconnectOnFailure = false; - producerSettings.Topic = "topic1"; - producerSettings.SourceId = "123"; - producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - producerSettings.Codec = ECodec::LZOP; - return producerSettings; - } - - Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - const size_t partitions = 1; - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - - testServer.WaitInit("topic1"); - - auto producer = testServer.PQLib->CreateProducer(MakeProducerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false); - UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); - auto isDead = producer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - auto write1 = producer->Write(1, TString("blob1")); - auto write2 = producer->Write(2, TString("blob2")); - auto write3 = producer->Write(3, TString("blob3")); - auto write4 = producer->Write(4, TString("blob4")); - auto write5 = producer->Write(5, TString("blob5")); - - testServer.PQLib = nullptr; - Cerr << "PQLib destroyed" << Endl; - - UNIT_ASSERT(write1.HasValue()); - UNIT_ASSERT(write2.HasValue()); - UNIT_ASSERT(write3.HasValue()); - UNIT_ASSERT(write4.HasValue()); - UNIT_ASSERT(write5.HasValue()); - - UNIT_ASSERT(write1.GetValue().Response.HasError()); - UNIT_ASSERT(write2.GetValue().Response.HasError()); - UNIT_ASSERT(write3.GetValue().Response.HasError()); - UNIT_ASSERT(write4.GetValue().Response.HasError()); - UNIT_ASSERT(write5.GetValue().Response.HasError()); - - auto write6 = producer->Write(6, TString("blob6")); - UNIT_ASSERT(write6.HasValue()); - UNIT_ASSERT(write6.GetValue().Response.HasError()); - } - - Y_UNIT_TEST(WriteToDeadProducer) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLibSettings pqLibSettings; - pqLibSettings.DefaultLogger = logger; - THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); - - auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, false); - auto f = producer->Start(); - UNIT_ASSERT(f.GetValueSync().Response.HasError()); - Cerr << f.GetValueSync().Response << "\n"; - auto isDead = producer->IsDead(); - isDead.Wait(); - UNIT_ASSERT(isDead.HasValue()); - - auto write = producer->Write(1, TString("blob")); - - Cerr << write.GetValueSync().Response << "\n"; - UNIT_ASSERT(write.GetValueSync().Response.HasError()); - UNIT_ASSERT_STRINGS_EQUAL(write.GetValueSync().Response.GetError().GetDescription(), "Destroyed"); - } - - Y_UNIT_TEST(Auth_StartProducerWithInvalidTokenFromGrpcMetadataPointOfView_StartFailedAndProduerDiedWithBadRequestErrors) { - TPQLibSettings pqLibSettings; - if (!std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED")) - return; - - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(); - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - pqLibSettings.DefaultLogger = logger; - THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); - - auto fillCredentialsWithInvalidToken = [](NPersQueue::TCredentials* authInfo) { - authInfo->set_tvm_service_ticket("token\n"); - }; - auto producerSettings = MakeProducerSettings(testServer); - producerSettings.CredentialsProvider = std::make_shared<TCallbackCredentialsProvider>(std::move(fillCredentialsWithInvalidToken)); - auto producer = PQLib->CreateProducer(producerSettings, logger, false); - - - Cerr << "Wait for producer start procedure end" << Endl; - auto startResponse = producer->Start().GetValueSync().Response; - - UNIT_ASSERT(startResponse.has_error()); - UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::ERROR, startResponse.error().code(), startResponse); - - Cerr << "Wait for producer death" << Endl; - auto deathCause = producer->IsDead().GetValueSync(); - - - UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::ERROR, deathCause.code(), deathCause); - } - - Y_UNIT_TEST(Codecs_WriteWithNonDefaultCodecThatRequiresAdditionalConfiguration_ConsumerDiesWithBadRequestError) { - SDKTestSetup setup{"Codecs_WriteWithNonDefaultCodecThatRequiresAdditionalConfiguration_ConsumerIsDeadWithBadRequestError"}; - auto log = setup.GetLog(); - auto producerSettings = setup.GetProducerSettings(); - // TTestServer::AnnoyingClient creates topic with default codecs set: raw, gzip, lzop. zstd not included - producerSettings.Codec = ECodec::ZSTD; - auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); - log << TLOG_INFO << "Wait for producer start"; - auto startResponse = producer->Start().GetValueSync().Response; - UNIT_ASSERT_C(!startResponse.HasError(), startResponse); - - log << TLOG_INFO << "Wait for write response"; - auto writeResponse = producer->Write(NUnitTest::RandomString(250 * 1024, std::rand())).GetValueSync().Response; - - - UNIT_ASSERT(writeResponse.HasError()); - Cerr << writeResponse << "\n"; - UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::BAD_REQUEST, writeResponse.GetError().GetCode(), writeResponse); - } - - void StartProducerWithDiscovery(bool cds, const TString& preferredCluster = {}, bool addBrokenDatacenter = !GrpcV1EnabledByDefault()) { - SDKTestSetup setup{"StartProducerWithDiscovery", false}; - setup.Start(true, addBrokenDatacenter); - auto log = setup.GetLog(); - auto producerSettings = setup.GetProducerSettings(); - if (preferredCluster) { - producerSettings.PreferredCluster = preferredCluster; - } - auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); - auto startResponse = producer->Start().GetValueSync().Response; - UNIT_ASSERT_C(!startResponse.HasError(), startResponse); - - log << TLOG_INFO << "Wait for write response"; - auto writeResponse = producer->Write(1, TString("blob1")).GetValueSync().Response; - UNIT_ASSERT(!writeResponse.HasError()); - - producerSettings.Server.UseLogbrokerCDS = cds ? EClusterDiscoveryUsageMode::Use : EClusterDiscoveryUsageMode::DontUse; - producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); - startResponse = producer->Start().GetValueSync().Response; - UNIT_ASSERT_C(!startResponse.HasError(), startResponse); - - log << TLOG_INFO << "Wait for write response"; - writeResponse = producer->Write(2, TString("blob2")).GetValueSync().Response; - UNIT_ASSERT(!writeResponse.HasError()); - } - - Y_UNIT_TEST(StartProducerWithCDS) { - StartProducerWithDiscovery(true); - } - - Y_UNIT_TEST(StartProducerWithoutCDS) { - StartProducerWithDiscovery(false); - } - - Y_UNIT_TEST(StartProducerWithCDSAndPreferAvailableCluster) { - StartProducerWithDiscovery(true, "dc1"); - } - - Y_UNIT_TEST(StartProducerWithCDSAndPreferUnavailableCluster) { - StartProducerWithDiscovery(true, "dc2", true); - } - - Y_UNIT_TEST(StartProducerWithCDSAndPreferUnknownCluster) { - SDKTestSetup setup{"PreferUnknownCluster"}; - auto producerSettings = setup.GetProducerSettings(); - producerSettings.Server.UseLogbrokerCDS = EClusterDiscoveryUsageMode::Use; - producerSettings.PreferredCluster = "blablabla"; - auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); - auto startResponse = producer->Start().GetValueSync().Response; - UNIT_ASSERT_C(startResponse.HasError(), startResponse); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp deleted file mode 100644 index 6e1c3f9b2ec..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "queue_pool.h" - -#include <util/digest/city.h> - -namespace NPersQueue { -void TQueuePool::Start(size_t queuesCount) { - Y_VERIFY(queuesCount > 0); - Queues.resize(queuesCount); - for (auto& queuePtr : Queues) { - queuePtr = std::make_shared<TThreadPool>(); - queuePtr->Start(1); // start one thread for each tag - } -} - -void TQueuePool::Stop() { - for (auto& queuePtr : Queues) { - queuePtr->Stop(); - } - Queues.clear(); -} - -const std::shared_ptr<IThreadPool>& TQueuePool::GetQueuePtr(const void* tag) { - Y_VERIFY(!Queues.empty()); - const size_t queue = static_cast<size_t>(CityHash64(reinterpret_cast<const char*>(&tag), sizeof(tag))) % Queues.size(); - return Queues[queue]; -} - -IThreadPool& TQueuePool::GetQueue(const void* tag) { - return *GetQueuePtr(tag); -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h deleted file mode 100644 index 961b8ff4464..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once -#include <util/generic/ptr.h> -#include <util/thread/pool.h> - -#include <functional> -#include <memory> -#include <vector> - -namespace NPersQueue { -class TQueuePool { -public: - void Start(size_t queuesCount); - void Stop(); - - // get one-thread-queue for processing specified tag (address) - IThreadPool& GetQueue(const void* tag); - - const std::shared_ptr<IThreadPool>& GetQueuePtr(const void* tag); - -private: - std::vector<std::shared_ptr<IThreadPool>> Queues; -}; -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp deleted file mode 100644 index 9f8d792e52b..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "queue_pool.h" - -#include <library/cpp/testing/unittest/registar.h> - -#include <unordered_map> - -namespace NPersQueue { -Y_UNIT_TEST_SUITE(TQueuePoolTest) { - Y_UNIT_TEST(QueuesDistribution) { - TQueuePool pool; - pool.Start(10); - size_t addresses[10000] = {}; - std::unordered_map<IThreadPool*, size_t> queueAddresses; - for (size_t& address : addresses) { - IThreadPool* q = &pool.GetQueue(&address); - ++queueAddresses[q]; - UNIT_ASSERT_EQUAL(q, &pool.GetQueue(&address)); // one address always leads to one queue - } - UNIT_ASSERT_VALUES_EQUAL(queueAddresses.size(), 10); - for (const auto& queueToCount : queueAddresses) { - UNIT_ASSERT_C(queueToCount.second >= 850, "Count: " << queueToCount.second); - UNIT_ASSERT_C(queueToCount.second <= 1150, "Count: " << queueToCount.second); - } - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp deleted file mode 100644 index 02b3aafafc7..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp +++ /dev/null @@ -1,526 +0,0 @@ -#include "retrying_consumer.h" - -namespace { - bool IsRetryable(const NPersQueue::TError& err) { - switch (err.code()) { - case NPersQueue::NErrorCode::INITIALIZING: - case NPersQueue::NErrorCode::OVERLOAD: - case NPersQueue::NErrorCode::READ_TIMEOUT: - case NPersQueue::NErrorCode::TABLET_IS_DROPPED: - case NPersQueue::NErrorCode::CREATE_TIMEOUT: - case NPersQueue::NErrorCode::ERROR: - case NPersQueue::NErrorCode::CLUSTER_DISABLED: - return true; - default: - return false; - } - } -} - -namespace NPersQueue { - -TRetryingConsumer::TRetryingConsumer(const TConsumerSettings& settings, - std::shared_ptr<void> destroyEventRef, - TIntrusivePtr<TPQLibPrivate> pqLib, - TIntrusivePtr<ILogger> logger) - : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) - , Settings(settings) - , Logger(std::move(logger)) - , IsDeadPromise(NThreading::NewPromise<TError>()) - , ConsumerDestroyedPromise(NThreading::NewPromise<void>()) - , GenCounter(1) - , CookieCounter(1) - , ReconnectionAttemptsDone(0) - , Stopping(false) - , Reconnecting(false) -{ - Y_VERIFY(Settings.ReconnectOnFailure, "ReconnectOnFailure should be set."); - Y_ENSURE(Settings.MaxAttempts != 0, "MaxAttempts setting can't be zero."); - Y_ENSURE(Settings.ReconnectionDelay != TDuration::Zero(), "ReconnectionDelay setting can't be zero."); - Y_ENSURE(Settings.MaxReconnectionDelay != TDuration::Zero(), "MaxReconnectionDelay setting can't be zero."); - Y_ENSURE(Settings.StartSessionTimeout != TDuration::Zero(), "StartSessionTimeout setting can't be zero."); - - Settings.ReconnectOnFailure = false; // reset flag, to use the settings for creating sub consumer -} - -TRetryingConsumer::~TRetryingConsumer() noexcept { - Cancel(); -} - -NThreading::TFuture<TConsumerCreateResponse> TRetryingConsumer::Start(TInstant deadline) noexcept { - StartPromise = NThreading::NewPromise<TConsumerCreateResponse>(); - if (deadline != TInstant::Max()) { - std::weak_ptr<TRetryingConsumer> weakRef = shared_from_this(); - auto onDeadline = [weakRef] { - auto self = weakRef.lock(); - if (nullptr != self) { - self->OnStartDeadline(); - } - }; - PQLib->GetScheduler().Schedule(deadline, this, onDeadline); - } - DoReconnect(deadline); - return StartPromise.GetFuture(); -} - -NThreading::TFuture<TError> TRetryingConsumer::IsDead() noexcept { - return IsDeadPromise.GetFuture(); -} - -void TRetryingConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { - if (!StartPromise.Initialized() || Stopping) { - TReadResponse res; - res.MutableError()->SetDescription("consumer is not ready"); - res.MutableError()->SetCode(NErrorCode::ERROR); - promise.SetValue(TConsumerMessage{std::move(res)}); - return; - } - - if (!ReadyResponses.empty()) { - Y_ASSERT(PendingRequests.empty()); - promise.SetValue(std::move(ReadyResponses.front())); - ReadyResponses.pop_front(); - return; - } - - PendingRequests.push_back(promise); - - if (Consumer && StartFuture.HasValue()) { - DoRequest(); - } -} - -static void FormatCookies(TStringBuilder& ret, const TVector<ui64>& cookies) { - ret << "{"; - for (size_t i = 0; i < cookies.size(); ++i) { - if (i > 0) { - ret << ", "; - } - ret << cookies[i]; - } - ret << "}"; -} - -static TString FormatCommitForLog(const TStringBuf reason, const TVector<ui64>& cookies, const TVector<ui64>& originalCookies, const TVector<ui64>& committedCookies) { - TStringBuilder ret; - ret << "Commit cookies by retrying consumer" << reason << ". User cookies: "; - FormatCookies(ret, cookies); - ret << ". Subconsumer cookies to commit: "; - FormatCookies(ret, originalCookies); - ret << ". Skipped cookies: "; - FormatCookies(ret, committedCookies); - return std::move(ret); -} - -void TRetryingConsumer::Commit(const TVector<ui64>& cookies) noexcept { - if (!Consumer) { - if (!StartPromise.Initialized() || Stopping) { - Destroy("Not ready", NErrorCode::BAD_REQUEST); - } else { - // just response that cookies were commited - FastCommit(cookies); - DEBUG_LOG(FormatCommitForLog(". Consumer is not initialied", cookies, {}, cookies), "", SessionId); - } - return; - } - if (Settings.CommitsDisabled) { - Destroy("Commits are disabled", NErrorCode::BAD_REQUEST); - return; - } - - TVector<ui64> originalCookies(Reserve(cookies.size())); - // cookies which can be treated as committed - TVector<ui64> commitedCookies; - ui64 minCookie = CookieCounter - Cookies.size(); - for (auto cookie : cookies) { - if (cookie >= minCookie && cookie < CookieCounter) { - auto& cookieInfo = Cookies[cookie - minCookie]; - Y_VERIFY(cookieInfo.UserCookie == cookie); - if (0 != cookieInfo.OriginalCookie) { - CommittingCookies[cookieInfo.OriginalCookie] = cookie; - originalCookies.push_back(cookieInfo.OriginalCookie); - cookieInfo.OriginalCookie = 0; - for (auto* lock : cookieInfo.Locks) { - lock->Cookies.erase(cookie); - } - } else { - commitedCookies.push_back(cookie); - } - } else if (cookie >= CookieCounter) { - Destroy("Unknown cookie", NErrorCode::BAD_REQUEST); - break; - } else { - commitedCookies.push_back(cookie); - } - } - if (!originalCookies.empty()) { - Consumer->Commit(originalCookies); - } - if (!commitedCookies.empty()) { - FastCommit(commitedCookies); - } - DEBUG_LOG(FormatCommitForLog("", cookies, originalCookies, commitedCookies), "", SessionId); - // clean commited cookies - while (!Cookies.empty() && 0 == Cookies.front().OriginalCookie) { - Cookies.pop_front(); - } -} - -void TRetryingConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { - if (!Consumer) { - // client should receive message about release lock - return; - } - auto lockIt = Locks.find(std::make_pair(topic, partition)); - if (lockIt == Locks.end() || !lockIt->second.Locked) { - WARN_LOG("Requesting partition status on partition without lock. Topic: " << topic << ". Partition: " << partition, "", SessionId); - } else if (lockIt->second.Gen != generation) { - WARN_LOG("Requesting partition status on partition with wrong generation of lock. Topic: " << topic << ". Partition: " << partition << ". Generation: " << generation, "", SessionId); - } else { - Consumer->RequestPartitionStatus(topic, partition, lockIt->second.OriginalGen); - } -} - -void TRetryingConsumer::Cancel() { - Destroy(GetCancelReason()); -} - -NThreading::TFuture<void> TRetryingConsumer::Destroyed() noexcept { - return ConsumerDestroyedPromise.GetFuture(); -} - -void TRetryingConsumer::OnStartDeadline() { - if (!StartPromise.HasValue()) { - TError error; - error.SetDescription("Start timeout."); - error.SetCode(NErrorCode::CREATE_TIMEOUT); - Destroy(error); - } -} - -void TRetryingConsumer::OnConsumerDead(const TError& error) { - WARN_LOG("Subconsumer is dead: " << error, "", SessionId); - ScheduleReconnect(); -} - -void TRetryingConsumer::ScheduleReconnect() { - if (Stopping || Reconnecting) { - return; - } - Reconnecting = true; - if (!Locks.empty()) { - // need to notify client that all locks are expired - for (auto&& [key, lock] : Locks) { - if (lock.Locked) { - TReadResponse response; - auto* release = response.MutableRelease(); - release->set_generation(lock.Gen); - release->set_topic(key.first); - release->set_partition(key.second); - release->set_can_commit(false); - FastResponse(TConsumerMessage(std::move(response))); - } - } - } - if (!CommittingCookies.empty()) { - // need to notify client that all cookies are commited, because these cookies aren't valid anymore - TReadResponse response; - auto* cookies = response.MutableCommit()->MutableCookie(); - cookies->Reserve(CommittingCookies.size()); - for (const auto& cookiePair : CommittingCookies) { - cookies->Add(cookiePair.second); - } - FastResponse(TConsumerMessage(std::move(response))); - } - - Cookies.clear(); - CommittingCookies.clear(); - Locks.clear(); - Consumer = nullptr; - - if (ReconnectionAttemptsDone >= Settings.MaxAttempts) { - Destroy(TStringBuilder() << "Failed " << ReconnectionAttemptsDone << " reconnection attempts"); - return; - } - - ++ReconnectionAttemptsDone; - - TDuration delay = Min(Settings.MaxReconnectionDelay, ReconnectionAttemptsDone * Settings.ReconnectionDelay); - std::weak_ptr<TRetryingConsumer> weakRef = shared_from_this(); - PQLib->GetScheduler().Schedule(delay, this, [weakRef]() { - auto self = weakRef.lock(); - if (nullptr != self) { - self->DoReconnect(self->Settings.StartSessionTimeout.ToDeadLine()); - self->Reconnecting = false; - } - }); -} - -void TRetryingConsumer::DoReconnect(TInstant deadline) { - if (Stopping) { - return; - } - DEBUG_LOG("Create subconsumer", "", SessionId); - Consumer = PQLib->CreateRawConsumer(Settings, DestroyEventRef, Logger); - std::weak_ptr<TRetryingConsumer> weak(shared_from_this()); - StartFuture = Consumer->Start(deadline); - PQLib->Subscribe(StartFuture, this, [weak](const NThreading::TFuture<TConsumerCreateResponse>& f) { - auto self = weak.lock(); - if (nullptr != self) { - self->StartProcessing(f); - } - }); - - PQLib->Subscribe(Consumer->IsDead(), this, [weak](const NThreading::TFuture<TError>& error) { - auto self = weak.lock(); - if (nullptr != self) { - self->OnConsumerDead(error.GetValue()); - } - }); -} - -void TRetryingConsumer::StartProcessing(const NThreading::TFuture<TConsumerCreateResponse>& f) { - Y_VERIFY(f.HasValue()); - if (f.GetValue().Response.HasError()) { - WARN_LOG("Cannot create subconsumer: " << f.GetValue().Response.GetError(), "", SessionId); - if (IsRetryable(f.GetValue().Response.GetError())) { - ScheduleReconnect(); - } else { - Destroy(f.GetValue().Response.GetError()); - } - } else { - if (!SessionId) { - SessionId = f.GetValue().Response.GetInit().GetSessionId(); - } - ReconnectionAttemptsDone = 0; - // need to schedule again all pending requests which are running before connection lost - for (size_t cnt = PendingRequests.size(); 0 != cnt; --cnt) { - DoRequest(); - } - StartPromise.TrySetValue(f.GetValue()); - } -} - -void TRetryingConsumer::SubscribeDestroyed() { - NThreading::TPromise<void> promise = ConsumerDestroyedPromise; - auto handler = [promise](const auto&) mutable { - promise.SetValue(); - }; - if (Consumer) { - WaitExceptionOrAll(DestroyedPromise.GetFuture(), Consumer->Destroyed()).Subscribe(handler); - } else { - DestroyedPromise.GetFuture().Subscribe(handler); - } - - DestroyPQLibRef(); -} - -void TRetryingConsumer::Destroy(const TError& error) { - if (Stopping) { - return; - } - - Stopping = true; - - SubscribeDestroyed(); - - TConsumerMessage message = [&]() { - TReadResponse response; - *response.MutableError() = error; - return TConsumerMessage(std::move(response)); - }(); - - for (auto& r : PendingRequests) { - r.TrySetValue(message); - } - IsDeadPromise.TrySetValue(error); - if (StartPromise.Initialized()) { - StartPromise.TrySetValue(TConsumerCreateResponse(std::move(message.Response))); - } - PendingRequests.clear(); - ReadyResponses.clear(); - Cookies.clear(); - CommittingCookies.clear(); - Locks.clear(); - Consumer = nullptr; - Reconnecting = false; -} - -void TRetryingConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) { - TError error; - error.SetDescription(description); - error.SetCode(code); - Destroy(error); -} - -void TRetryingConsumer::DoRequest() { - Y_VERIFY(Consumer); - std::weak_ptr<TRetryingConsumer> weak(shared_from_this()); - PQLib->Subscribe(Consumer->GetNextMessage(), this, [weak](NThreading::TFuture<TConsumerMessage>& f) mutable { - auto self = weak.lock(); - if (nullptr != self) { - self->ProcessResponse(f.ExtractValueSync()); - } - }); -} - -void TRetryingConsumer::ProcessResponse(TConsumerMessage&& message) { - switch (message.Type) { - case NPersQueue::EMessageType::EMT_DATA: - { - auto* data = message.Response.mutable_data(); - TCookieInfo* cookie = nullptr; - if (!Settings.CommitsDisabled) { - Cookies.push_back(TCookieInfo{{}, data->cookie(), CookieCounter++}); - cookie = &Cookies.back(); - data->set_cookie(cookie->UserCookie); - DEBUG_LOG("Got data from subconsumer. Cookie: " << cookie->OriginalCookie << ". User cookie: " << cookie->UserCookie, "", SessionId); - } - - for (auto& b : data->message_batch()) { - auto lockIt = Locks.find(std::make_pair(b.topic(), b.partition())); - if (lockIt != Locks.end()) { - Y_VERIFY(lockIt->second.Locked); - if (!Settings.CommitsDisabled) { - Y_ASSERT(nullptr != cookie); - cookie->Locks.emplace(&lockIt->second); - lockIt->second.Cookies.emplace(cookie->UserCookie); - } - - // Validate that all offsets are >= LockInfo.ReadOffset as expected. - if (lockIt->second.ReadOffset != 0) { - for (const auto& message : b.message()) { - if (message.offset() < lockIt->second.ReadOffset) { - Destroy( - TStringBuilder() - << "Fatal error: expected offsets for topic " << b.topic() - << " partition " << b.partition() << " >= " << lockIt->second.ReadOffset - << ", but got offset " << message.offset() - ); - return; - } - } - } - } - } - // TODO (bulatman) what about batched_data? - } - break; - case NPersQueue::EMessageType::EMT_LOCK: - { - auto* lock = message.Response.mutable_lock(); - auto& lockInfo = Locks[std::make_pair(lock->topic(), lock->partition())]; - Y_VERIFY(!lockInfo.Locked); - lockInfo.Locked = true; - lockInfo.Gen = GenCounter++; - lockInfo.OriginalGen = lock->generation(); - - lock->set_generation(lockInfo.Gen); - - DEBUG_LOG("Got lock from subconsumer on (" << lock->topic() << ", " << lock->partition() << "). Generation: " << lockInfo.OriginalGen << ". User generation: " << lockInfo.Gen, "", SessionId); - - std::weak_ptr<TRetryingConsumer> weak = shared_from_this(); - PQLib->Subscribe(message.ReadyToRead.GetFuture(), this, - [weak, topic = lock->topic(), partition = lock->partition(), generation = lock->generation()](const NThreading::TFuture<NPersQueue::TLockInfo>& lockInfo) { - auto self = weak.lock(); - if (nullptr != self) { - self->UpdateReadyToRead(lockInfo.GetValue(), topic, partition, generation); - } - }); - } - break; - case NPersQueue::EMessageType::EMT_RELEASE: - { - auto* release = message.Response.mutable_release(); - const bool softRelease = release->can_commit(); - auto lockIt = Locks.find(std::make_pair(release->topic(), release->partition())); - const bool lockInfoFound = lockIt != Locks.end(); - DEBUG_LOG("Got release from subconsumer on (" << release->topic() << ", " << release->partition() << "). Can commit: " << softRelease << ". Has lock info: " << lockInfoFound << ". Generation: " << release->generation() << ". User generation: " << (lockInfoFound ? lockIt->second.Gen : 0), "", SessionId); - if (lockInfoFound) { // It is normal situation when client receives Release(canCommit=true) and then Release(canCommit=false). - auto& lockInfo = lockIt->second; - Y_VERIFY(lockInfo.OriginalGen == release->generation(), "lock generation mismatch"); - release->set_generation(lockInfo.Gen); - - if (softRelease) { - lockInfo.Locked = false; - } else { - for (auto cookie : lockInfo.Cookies) { - auto& cookieInfo = Cookies[cookie + Cookies.size() - CookieCounter]; - Y_VERIFY(cookieInfo.UserCookie == cookie); - // Lock is not valid anymore. - cookieInfo.Locks.erase(&lockInfo); - } - Locks.erase(lockIt); - } - } else { - return; - } - } - break; - case NPersQueue::EMessageType::EMT_STATUS: - { - auto* status = message.Response.mutable_partition_status(); - auto lockIt = Locks.find(std::make_pair(status->topic(), status->partition())); - Y_VERIFY(lockIt != Locks.end() && lockIt->second.Locked && lockIt->second.OriginalGen == status->generation()); - status->set_generation(lockIt->second.Gen); - } - break; - case NPersQueue::EMessageType::EMT_COMMIT: - { - auto* cookies = message.Response.mutable_commit()->mutable_cookie(); - // convert cookies - for (int i = 0; i < cookies->size(); ++i) { - auto it = CommittingCookies.find(cookies->Get(i)); - Y_VERIFY(it != CommittingCookies.end(), "unknown commited cookie!"); - cookies->Set(i, it->second); - CommittingCookies.erase(it); - } - } - break; - - case NPersQueue::EMessageType::EMT_ERROR: - // check error, if retryable, need to recreate consumer and read message again - const bool retryable = IsRetryable(message.Response.error()); - DEBUG_LOG("Got error from subconsumer: " << message.Response.error() << ". Retryable: " << retryable, "", SessionId); - if (retryable) { - ScheduleReconnect(); - return; - } - break; - }; - - if (!PendingRequests.empty()) { - PendingRequests.front().SetValue(std::move(message)); - PendingRequests.pop_front(); - } else { - ReadyResponses.push_back(std::move(message)); - } -} - -void TRetryingConsumer::FastResponse(TConsumerMessage&& message) { - if (!PendingRequests.empty()) { - PendingRequests.front().SetValue(std::move(message)); - PendingRequests.pop_front(); - } else { - ReadyResponses.push_back(std::move(message)); - } -} - -void TRetryingConsumer::FastCommit(const TVector<ui64>& cookies) { - TReadResponse response; - auto* cookiesMessage = response.MutableCommit()->MutableCookie(); - cookiesMessage->Reserve(cookies.size()); - for (auto cookie : cookies) { - cookiesMessage->Add(cookie); - } - FastResponse(TConsumerMessage(std::move(response))); -} - -void TRetryingConsumer::UpdateReadyToRead(const NPersQueue::TLockInfo& readyToRead, const TString& topic, ui32 partition, ui64 generation) { - const auto lockIt = Locks.find(std::make_pair(topic, partition)); - if (lockIt != Locks.end() && lockIt->second.Locked && lockIt->second.Gen == generation) { - lockIt->second.ReadOffset = readyToRead.ReadOffset; - } -} - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h deleted file mode 100644 index 448d35e4c80..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h +++ /dev/null @@ -1,107 +0,0 @@ -#pragma once - -#include "consumer.h" -#include "scheduler.h" -#include "internals.h" -#include "persqueue_p.h" -#include "iconsumer_p.h" - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> - -#include <library/cpp/threading/future/future.h> - -#include <util/generic/hash.h> -#include <util/generic/hash_set.h> -#include <util/generic/vector.h> - -#include <deque> - -namespace NPersQueue { - -// @brief consumer implementation which transparently retries all connectivity errors -class TRetryingConsumer: public IConsumerImpl, public std::enable_shared_from_this<TRetryingConsumer> { - // @brief locked partitions info - struct TLockInfo { - THashSet<ui64> Cookies; // related cookies - ui64 Gen = 0; - ui64 OriginalGen = 0; // original generation - ui64 ReadOffset = 0; // read offset specified by client - bool Locked = false; - }; - - struct TCookieInfo { - THashSet<TLockInfo*> Locks; // related locks - ui64 OriginalCookie = 0; // zero means invalid cookie - ui64 UserCookie = 0; - }; - -public: - TRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, - TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); - - ~TRetryingConsumer() noexcept override; - - NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept override; - - using IConsumerImpl::GetNextMessage; - void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; - - void Commit(const TVector<ui64>& cookies) noexcept override; - - void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; - - NThreading::TFuture<TError> IsDead() noexcept override; - - void Cancel() override; - - NThreading::TFuture<void> Destroyed() noexcept override; - -private: - void OnStartDeadline(); - void OnConsumerDead(const TError& error); - void ScheduleReconnect(); - void DoReconnect(TInstant deadline); - void StartProcessing(const NThreading::TFuture<TConsumerCreateResponse>& f); - void SubscribeDestroyed(); - void Destroy(const TError& error); - void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); - void DoRequest(); - void ProcessResponse(TConsumerMessage&& message); - void FastResponse(TConsumerMessage&& message); - void FastCommit(const TVector<ui64>& cookies); - void UpdateReadyToRead(const NPersQueue::TLockInfo& readyToRead, const TString& topic, ui32 partition, ui64 generation); - -private: - TConsumerSettings Settings; - TIntrusivePtr<ILogger> Logger; - std::shared_ptr<IConsumerImpl> Consumer; - TString SessionId; - - NThreading::TFuture<TConsumerCreateResponse> StartFuture; - NThreading::TPromise<TConsumerCreateResponse> StartPromise; - NThreading::TPromise<TError> IsDeadPromise; - NThreading::TPromise<void> ConsumerDestroyedPromise; - - // requests which are waiting for response - std::deque<NThreading::TPromise<TConsumerMessage>> PendingRequests; - // ready messages for returning to clients immediately - std::deque<TConsumerMessage> ReadyResponses; - // active cookies - std::deque<TCookieInfo> Cookies; - THashMap<ui64, ui64> CommittingCookies; - // active data per topic, partition - THashMap<std::pair<TString, ui64>, TLockInfo> Locks; // topic, partition -> LockInfo - - // number of unsuccessful retries after last error - ui64 GenCounter; - ui64 CookieCounter; - unsigned ReconnectionAttemptsDone; - // destroying process started - bool Stopping; - // reconnecting in process - bool Reconnecting; -}; -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp deleted file mode 100644 index 8065ed717ac..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp +++ /dev/null @@ -1,604 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/guid.h> - -using namespace NThreading; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; - -namespace NPersQueue { - Y_UNIT_TEST_SUITE(TRetryingConsumerTest) { - static TConsumerSettings FakeSettings() { - TConsumerSettings settings; - settings.ReconnectOnFailure = true; - settings.Server = TServerSetting{"localhost"}; - settings.Topics.push_back("topic"); - settings.ClientId = "client"; - return settings; - } - - Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) { - // Test that consumer doesn't hang on till shutdown - TPQLib lib; - auto settings = FakeSettings(); - lib.CreateConsumer(settings); - } - - Y_UNIT_TEST(StartDeadlineExpires) { - TPQLibSettings libSettings; - libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); - libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG); - TPQLib lib(libSettings); - THolder<IConsumer> consumer; - { - auto settings = FakeSettings(); - settings.ReconnectionDelay = TDuration::MilliSeconds(1); - settings.StartSessionTimeout = TDuration::MilliSeconds(10); - if (GrpcV1EnabledByDefault()) { - settings.MaxAttempts = 3; - } - consumer = lib.CreateConsumer(settings); - } - const TInstant beforeStart = TInstant::Now(); - auto future = consumer->Start(TDuration::MilliSeconds(100)); - if (!GrpcV1EnabledByDefault()) { - UNIT_ASSERT(future.GetValueSync().Response.HasError()); - UNIT_ASSERT_EQUAL_C(future.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, - "Error: " << future.GetValueSync().Response.GetError()); - const TInstant now = TInstant::Now(); - UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(100), now); - } - auto isDead = consumer->IsDead(); - isDead.Wait(); - - DestroyAndWait(consumer); - } - - Y_UNIT_TEST(StartMaxAttemptsExpire) { - TPQLibSettings libSettings; - libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); - libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG); - TPQLib lib(libSettings); - THolder<IConsumer> consumer; - { - auto settings = FakeSettings(); - settings.ReconnectionDelay = TDuration::MilliSeconds(10); - settings.StartSessionTimeout = TDuration::MilliSeconds(10); - settings.MaxAttempts = 3; - consumer = lib.CreateConsumer(settings); - } - const TInstant beforeStart = TInstant::Now(); - auto future = consumer->Start(); - if (!GrpcV1EnabledByDefault()) { - UNIT_ASSERT(future.GetValueSync().Response.HasError()); - } - auto deadFuture = consumer->IsDead(); - deadFuture.Wait(); - const TInstant now = TInstant::Now(); - UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(30), now); - - DestroyAndWait(consumer); - } - - static void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { - const TProducerCommitResponse& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); - } - - static void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); - } - - static void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response); - } - - static void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response); - } - - static TProducerSettings MakeProducerSettings(const TTestServer& testServer) { - TProducerSettings producerSettings; - producerSettings.ReconnectOnFailure = false; - producerSettings.Topic = "topic1"; - producerSettings.SourceId = "123"; - producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - producerSettings.Codec = ECodec::LZOP; - return producerSettings; - } - - static TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { - TConsumerSettings settings; - settings.ReconnectOnFailure = true; - settings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - settings.Topics.emplace_back("topic1"); - settings.ClientId = "user"; - return settings; - } - - Y_UNIT_TEST(ReconnectsToServer) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - const size_t partitions = 10; - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - - testServer.WaitInit("topic1"); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLib PQLib; - - auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); - UNIT_ASSERT(!producer->IsDead().HasValue()); - producer->Start().Wait(); - - auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); - TFuture<TError> isDead = consumer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - consumer->Start().Wait(); - - auto read1 = consumer->GetNextMessage(); - - // write first value - auto write1 = producer->Write(1, TString("blob1")); - AssertWriteValid(write1); - AssertReadValid(read1); - - // commit - consumer->Commit({1}); - auto commitAck = consumer->GetNextMessage(); - Cerr << "wait for commit1\n"; - - AssertCommited(commitAck); - - Cerr << "After shutdown\n"; - testServer.ShutdownGRpc(); - - auto read2 = consumer->GetNextMessage(); - UNIT_ASSERT(!isDead.HasValue()); - testServer.EnableGRpc(); - - testServer.WaitInit("topic1"); - - Cerr << "Wait producer1 death\n"; - - producer->IsDead().Wait(); - - producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); - Cerr << "Wait producer2 start\n"; - producer->Start().Wait(); - UNIT_ASSERT(!producer->IsDead().HasValue()); - auto write2 = producer->Write(2, TString("blob2")); - Cerr << "Wait for second write\n"; - AssertWriteValid(write2); - Cerr << "wait for read second blob\n"; - AssertReadValid(read2); - UNIT_ASSERT_VALUES_EQUAL(2u, read2.GetValueSync().Response.GetData().GetCookie()); - UNIT_ASSERT(!isDead.HasValue()); - consumer->Commit({2}); - Cerr << "wait for commit2\n"; - auto commitAck2 = consumer->GetNextMessage(); - AssertCommited(commitAck2); - - DestroyAndWait(producer); - DestroyAndWait(consumer); - } - - static void DiesOnTooManyReconnectionAttempts(bool callRead) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2); - - testServer.WaitInit("topic1"); - - TConsumerSettings settings = MakeConsumerSettings(testServer); - settings.MaxAttempts = 3; - settings.ReconnectionDelay = TDuration::MilliSeconds(100); - auto consumer = testServer.PQLib->CreateConsumer(settings, testServer.PQLibSettings.DefaultLogger, false); - consumer->Start().Wait(); - TFuture<TError> isDead = consumer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - // shutdown server - const TInstant beforeShutdown = TInstant::Now(); - testServer.ShutdownServer(); - - NThreading::TFuture<TConsumerMessage> read; - if (callRead) { - read = consumer->GetNextMessage(); - } - - isDead.Wait(); - const TInstant afterDead = TInstant::Now(); - // 3 attempts: 100ms, 200ms and 300ms - UNIT_ASSERT_C(afterDead - beforeShutdown >= TDuration::MilliSeconds(GrpcV1EnabledByDefault() ? 300 : 600), "real difference: " << (afterDead - beforeShutdown)); - - if (callRead) { - AssertReadFailed(read); - } - } - - Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithoutRead) { - // Check that we reconnect even without explicit write errors - DiesOnTooManyReconnectionAttempts(false); - } - - Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithRead) { - DiesOnTooManyReconnectionAttempts(true); - } - - Y_UNIT_TEST(ReadBeforeConnectToServer) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - const size_t partitions = 10; - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - - testServer.WaitInit("topic1"); - - auto consumer = testServer.PQLib->CreateConsumer( - MakeConsumerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false - ); - auto consumerStarted = consumer->Start(); - auto read1 = consumer->GetNextMessage(); - consumerStarted.Wait(); - TVector<TFuture<TConsumerMessage>> messages; - for (int i = 2; i <= 5; ++i) { - messages.push_back(consumer->GetNextMessage()); - } - - auto producer = testServer.PQLib->CreateProducer( - MakeProducerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false - ); - producer->Start().Wait(); - - while (!messages.back().HasValue()) { - auto write = producer->Write(TString("data")); - AssertWriteValid(write); - } - - AssertReadValid(read1); - for (auto& msg: messages) { - AssertReadValid(msg); - } - - DestroyAndWait(producer); - DestroyAndWait(consumer); - } - - Y_UNIT_TEST(CancelsStartAfterPQLibDeath) { - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLibSettings pqLibSettings; - pqLibSettings.DefaultLogger = logger; - THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); - - auto consumer = PQLib->CreateConsumer(FakeSettings(), logger, false); - auto start = consumer->Start(); - if (!GrpcV1EnabledByDefault()) { - UNIT_ASSERT(!start.HasValue()); - } - - PQLib = nullptr; - - if (!GrpcV1EnabledByDefault()) { - UNIT_ASSERT(start.HasValue()); - UNIT_ASSERT(start.GetValue().Response.HasError()); - } - - auto dead = consumer->IsDead(); - if (GrpcV1EnabledByDefault()) { - dead.Wait(); - } else { - UNIT_ASSERT(dead.HasValue()); - } - - auto read = consumer->GetNextMessage(); - UNIT_ASSERT(read.HasValue()); - UNIT_ASSERT(read.GetValueSync().Response.HasError()); - } - - Y_UNIT_TEST(LockSession) { - SDKTestSetup setup("LockSession", false); - setup.GetGrpcServerOptions().SetGRpcShutdownDeadline(TDuration::MilliSeconds(100)); - setup.Start(); - auto consumerSettings = setup.GetConsumerSettings(); - consumerSettings.UseLockSession = true; - consumerSettings.ReconnectOnFailure = true; - auto consumer = setup.StartConsumer(consumerSettings); - - setup.WriteToTopic({"msg1", "msg2"}); - - size_t releases = 0; - for (size_t times = 0; times < 2; ++times) { - while (true) { - auto msgFuture = consumer->GetNextMessage(); - auto& msg = msgFuture.GetValueSync(); - setup.GetLog() << TLOG_INFO << "Got response from consumer: " << msg.Response; - if (times) { - UNIT_ASSERT_C(msg.Type == EMT_LOCK || msg.Type == EMT_DATA || msg.Type == EMT_RELEASE, msg.Response); - } else { - UNIT_ASSERT_C(msg.Type == EMT_LOCK || msg.Type == EMT_DATA, msg.Response); - } - if (msg.Type == EMT_LOCK) { - TLockInfo lockInfo; - lockInfo.ReadOffset = 1; - msg.ReadyToRead.SetValue(lockInfo); - } else if (msg.Type == EMT_DATA) { - UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().MessageBatchSize(), 1, msg.Response); - UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().GetMessageBatch(0).MessageSize(), 1, msg.Response); - UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg2", msg.Response); - break; - } else if (msg.Type == EMT_RELEASE) { - ++releases; - } - } - if (!times) { // force reconnect - setup.ShutdownGRpc(); - setup.EnableGRpc(); - } - } - UNIT_ASSERT_VALUES_EQUAL(releases, 1); - } - - Y_UNIT_TEST(ReadAhead) { - SDKTestSetup setup("ReadAhead"); - - auto consumerSettings = setup.GetConsumerSettings(); - consumerSettings.ReconnectOnFailure = true; - auto consumer = setup.StartConsumer(consumerSettings); - - setup.WriteToTopic({"msg"}); - - // Wait until read ahead occurs. - Sleep(TDuration::MilliSeconds(100)); - - setup.ReadFromTopic({{"msg"}}, true, consumer.Get()); - } - - Y_UNIT_TEST(ReadBeforeWrite) { - SDKTestSetup setup("ReadBeforeWrite"); - - auto consumerSettings = setup.GetConsumerSettings(); - consumerSettings.ReconnectOnFailure = true; - auto consumer = setup.StartConsumer(consumerSettings); - - auto msgFuture = consumer->GetNextMessage(); - - Sleep(TDuration::MilliSeconds(100)); - - setup.WriteToTopic({"msg"}); - - const auto& msg = msgFuture.GetValueSync(); - UNIT_ASSERT_EQUAL(msg.Type, EMT_DATA); - UNIT_ASSERT_STRINGS_EQUAL(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg"); - } - - Y_UNIT_TEST(CommitDisabled) { - SDKTestSetup setup("CommitDisabled"); - - auto consumerSettings = setup.GetConsumerSettings(); - consumerSettings.ReconnectOnFailure = true; - consumerSettings.CommitsDisabled = true; - auto consumer = setup.StartConsumer(consumerSettings); - setup.WriteToTopic({"msg", "msg2"}); - setup.ReadFromTopic({{"msg", "msg2"}}, false, consumer.Get()); - // check that commit fails with error when commits disabled - setup.WriteToTopic({"msg3"}); - auto msgFuture = consumer->GetNextMessage(); - const auto& msg = msgFuture.GetValueSync(); - UNIT_ASSERT_C(!msg.Response.HasError(), msg.Response); - UNIT_ASSERT_EQUAL(msg.Type, EMT_DATA); - UNIT_ASSERT_STRINGS_EQUAL(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg3"); - consumer->Commit({msg.Response.GetData().GetCookie()}); - auto isDead = consumer->IsDead(); - auto& err = isDead.GetValueSync(); - UNIT_ASSERT_STRINGS_EQUAL("Commits are disabled", err.GetDescription()); - } - - void LockReleaseTest(bool multiclusterConsumer, bool killPqrb) { - SDKTestSetup setup("LockRelease"); - const TString topicName = "lock_release"; - const size_t partitionsCount = 10; - setup.CreateTopic(topicName, setup.GetLocalCluster(), partitionsCount); - - auto consumerSettings = setup.GetConsumerSettings(); - consumerSettings.Topics[0] = topicName; - consumerSettings.ReconnectOnFailure = true; - consumerSettings.UseLockSession = true; - if (multiclusterConsumer) { // The same test for multicluster wrapper. - consumerSettings.ReadFromAllClusterSources = true; - consumerSettings.ReadMirroredPartitions = false; - } - auto consumer = setup.StartConsumer(consumerSettings); - - std::vector<int> partitionsLocked(partitionsCount * 2); // For both consumers. - std::vector<ui64> partitionsGens(partitionsCount * 2); - for (size_t i = 0; i < partitionsCount; ++i) { - auto lock = consumer->GetNextMessage().GetValueSync(); - UNIT_ASSERT_EQUAL_C(lock.Type, EMT_LOCK, lock.Response); - UNIT_ASSERT_C(!partitionsLocked[lock.Response.GetLock().GetPartition()], lock.Response); - partitionsLocked[lock.Response.GetLock().GetPartition()] = 1; - partitionsGens[lock.Response.GetLock().GetPartition()] = lock.Response.GetLock().GetGeneration(); - lock.ReadyToRead.SetValue({}); - } - - const size_t messagesCount = 100; - auto generateMessages = [&]() { - for (size_t i = 0; i < messagesCount; ++i) { // Write to several random partitions. - const TString uniqStr = CreateGuidAsString(); - auto settings = setup.GetProducerSettings(); - settings.Topic = topicName; - settings.SourceId = uniqStr; - auto producer = setup.StartProducer(settings); - setup.WriteToTopic({uniqStr}, producer.Get()); - } - }; - - generateMessages(); - - THolder<IConsumer> otherConsumer = nullptr; - bool rebalanced = false; - - NThreading::TFuture<TConsumerMessage> msg1 = consumer->GetNextMessage(); - NThreading::TFuture<TConsumerMessage> msg2; - size_t hardReleasesCount = 0; - - THashSet<TString> msgsReceived; - THashSet<ui64> cookiesToCommit; - auto locksCount = [&]() -> int { - return std::accumulate(partitionsLocked.begin(), partitionsLocked.end(), 0); - }; - auto allEventsReceived = [&]() -> bool { - return msgsReceived.size() >= messagesCount * 2 - && locksCount() == partitionsCount - && (!killPqrb || hardReleasesCount > 0) - && cookiesToCommit.empty(); - }; - - int iterationNum = 0; - while (!allEventsReceived()) { - ++iterationNum; - Cerr << "\nIterationNum: " << iterationNum << Endl; - Cerr << "msgsReceived.size(): " << msgsReceived.size() << Endl; - Cerr << "partitionsCount: " << partitionsCount << Endl; - Cerr << "locksGot: " << locksCount() << Endl; - Cerr << "hardReleasesCount: " << hardReleasesCount << Endl; - Cerr << "cookiesToCommit.size(): " << cookiesToCommit.size() << Endl; - auto waiter = msg2.Initialized() ? NThreading::WaitAny(msg1.IgnoreResult(), msg2.IgnoreResult()) : msg1.IgnoreResult(); - waiter.Wait(); - auto processValue = [&](auto& msg, auto& consumer) { - const bool isOtherConsumer = consumer == otherConsumer; - const ui64 consumerMix = isOtherConsumer ? 0x8000000000000000 : 0; - - switch (msg.GetValue().Type) { - case EMT_LOCK: - { - UNIT_ASSERT_LT(msg.GetValue().Response.GetLock().GetPartition(), partitionsCount); - const size_t idx = msg.GetValue().Response.GetLock().GetPartition() + (isOtherConsumer ? partitionsCount : 0); - if (msg.GetValue().Response.GetLock().GetGeneration() >= partitionsGens[idx]) { - UNIT_ASSERT_C(!partitionsLocked[idx], msg.GetValue().Response << ". Other: " << isOtherConsumer); - partitionsLocked[idx] = 1; - msg.GetValue().ReadyToRead.SetValue({}); - partitionsGens[idx] = msg.GetValue().Response.GetLock().GetGeneration(); - } - break; - } - case EMT_RELEASE: - { - UNIT_ASSERT_LT(msg.GetValue().Response.GetRelease().GetPartition(), partitionsCount); - const size_t idx = msg.GetValue().Response.GetRelease().GetPartition() + (isOtherConsumer ? partitionsCount : 0); - partitionsLocked[idx] = 0; - if (!msg.GetValue().Response.GetRelease().GetCanCommit()) { - ++hardReleasesCount; - } - if (!killPqrb) { - UNIT_ASSERT(msg.GetValue().Response.GetRelease().GetCanCommit()); // No restarts => soft release. - } - break; - } - case EMT_DATA: - { - const auto& resp = msg.GetValue().Response.GetData(); - UNIT_ASSERT(cookiesToCommit.insert(resp.GetCookie() | consumerMix).second); - for (const auto& batch : resp.GetMessageBatch()) { - for (const auto& message : batch.GetMessage()) { - msgsReceived.insert(message.GetData()); - } - } - consumer->Commit({resp.GetCookie()}); - break; - } - case EMT_ERROR: - case EMT_STATUS: - case EMT_COMMIT: - for (ui64 cookie : msg.GetValue().Response.GetCommit().GetCookie()) { - UNIT_ASSERT(cookiesToCommit.find(cookie | consumerMix) != cookiesToCommit.end()); - cookiesToCommit.erase(cookie | consumerMix); - } - break; - } - msg = consumer->GetNextMessage(); - }; - - if (msg1.HasValue()) { - processValue(msg1, consumer); - } - if (msg2.Initialized() && msg2.HasValue()) { - processValue(msg2, otherConsumer); - } - - if (!otherConsumer && msgsReceived.size() >= messagesCount / 4) { - otherConsumer = setup.StartConsumer(consumerSettings); - msg2 = otherConsumer->GetNextMessage(); - } - - if (!rebalanced && msgsReceived.size() >= messagesCount / 2) { - rebalanced = true; - - if (killPqrb) { - Sleep(TDuration::MilliSeconds(100)); - setup.KillPqrb(topicName, setup.GetLocalCluster()); - } - - generateMessages(); - } - } - - UNIT_ASSERT_VALUES_EQUAL(locksCount(), partitionsCount); - if (killPqrb) { - UNIT_ASSERT(hardReleasesCount > 0); - } else { - UNIT_ASSERT_VALUES_EQUAL(hardReleasesCount, 0); - } - } - - Y_UNIT_TEST(LockRelease) { - LockReleaseTest(false, false); - } - - Y_UNIT_TEST(LockReleaseHardRelease) { - LockReleaseTest(false, true); - } - - Y_UNIT_TEST(LockReleaseMulticluster) { - LockReleaseTest(true, false); - } - - Y_UNIT_TEST(LockReleaseMulticlusterHardRelease) { - LockReleaseTest(true, true); - } - - Y_UNIT_TEST(NonRetryableErrorOnStart) { - SDKTestSetup setup("NonRetryableErrorOnStart"); - - auto consumerSettings = setup.GetConsumerSettings(); - //consumerSettings. - consumerSettings.ReconnectOnFailure = true; - consumerSettings.Topics = { "unknown/topic" }; - THolder<IConsumer> consumer = setup.GetPQLib()->CreateConsumer(consumerSettings); - auto startFuture = consumer->Start(); - if (!GrpcV1EnabledByDefault()) { - Cerr << "===Got response: " << startFuture.GetValueSync().Response << Endl; - UNIT_ASSERT_C(startFuture.GetValueSync().Response.HasError(), startFuture.GetValueSync().Response); - UNIT_ASSERT_EQUAL(startFuture.GetValueSync().Response.GetError().GetCode(), NPersQueue::NErrorCode::UNKNOWN_TOPIC); - } - auto deadFuture = consumer->IsDead(); - UNIT_ASSERT_EQUAL_C(deadFuture.GetValueSync().GetCode(), NPersQueue::NErrorCode::UNKNOWN_TOPIC, deadFuture.GetValueSync()); - } - } -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp deleted file mode 100644 index bb562ea0ae2..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp +++ /dev/null @@ -1,370 +0,0 @@ -#include "retrying_producer.h" -#include "persqueue_p.h" - -#include <util/generic/strbuf.h> -#include <util/stream/zlib.h> -#include <util/string/cast.h> -#include <util/string/printf.h> -#include <util/string/vector.h> -#include <util/string/builder.h> - -namespace NPersQueue { - -TRetryingProducer::TRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> destroyEventRef, - TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) - : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) - , Settings(settings) - , Logger(std::move(logger)) - , IsDeadPromise(NThreading::NewPromise<TError>()) - , NeedRecreation(true) - , Stopping(false) - , ToProcess(0) - , LastReconnectionDelay(TDuration::Zero()) - , ReconnectionAttemptsDone(0) -{ - if (Settings.MaxAttempts == 0) { - ythrow yexception() << "MaxAttempts setting can't be zero."; - } - - if (Settings.ReconnectionDelay == TDuration::Zero()) { - ythrow yexception() << "ReconnectionDelay setting can't be zero."; - } - - if (Settings.StartSessionTimeout < PQLib->GetSettings().ChannelCreationTimeout) { - ythrow yexception() << "StartSessionTimeout can't be less than ChannelCreationTimeout."; - } -} - -TRetryingProducer::~TRetryingProducer() noexcept { - Destroy("Destructor called"); -} - -NThreading::TFuture<TProducerCreateResponse> TRetryingProducer::Start(TInstant deadline) noexcept { - StartPromise = NThreading::NewPromise<TProducerCreateResponse>(); - if (deadline != TInstant::Max()) { - std::weak_ptr<TRetryingProducer> self = shared_from_this(); - auto onDeadline = [self] { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnStartDeadline(); - } - }; - PQLib->GetScheduler().Schedule(deadline, this, onDeadline); - } - RecreateProducer(deadline); - return StartPromise.GetFuture(); -} - -void TRetryingProducer::RecreateProducer(TInstant deadline) noexcept { - Producer = nullptr; - if (Stopping) - return; - - Y_VERIFY(InFlightRequests.size() == Futures.size()); - DEBUG_LOG("Recreating subproducer. Futures size: " << Futures.size(), Settings.SourceId, ""); - if (Futures.empty()) { - DoRecreate(deadline); - } // otherwise it will be recreated in ProcessFutures() -} - -void TRetryingProducer::DoRecreate(TInstant deadline) noexcept { - Y_VERIFY(InFlightRequests.size() == Futures.size()); - Y_VERIFY(Futures.empty()); - if (Stopping) - return; - Y_VERIFY(NeedRecreation); - NeedRecreation = false; - if (ReconnectionAttemptsDone >= Settings.MaxAttempts) { - Destroy(TStringBuilder() << "Failed " << ReconnectionAttemptsDone << " reconnection attempts"); - return; - } - ++ReconnectionAttemptsDone; - DEBUG_LOG("Creating subproducer. Attempt: " << ReconnectionAttemptsDone, Settings.SourceId, ""); - Producer = PQLib->CreateRawProducer(Settings, DestroyEventRef, Logger); - StartFuture = Producer->Start(deadline); - - std::weak_ptr<TRetryingProducer> self(shared_from_this()); - PQLib->Subscribe(StartFuture, - this, - [self](const NThreading::TFuture<TProducerCreateResponse>& f) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->ProcessStart(f); - } - }); -} - -TDuration TRetryingProducer::UpdateReconnectionDelay() { - if (LastReconnectionDelay == TDuration::Zero()) { - LastReconnectionDelay = Settings.ReconnectionDelay; - } else { - LastReconnectionDelay *= 2; - } - LastReconnectionDelay = Min(LastReconnectionDelay, Settings.MaxReconnectionDelay); - return LastReconnectionDelay; -} - -void TRetryingProducer::ScheduleRecreation() { - if (Stopping) { - return; - } - NeedRecreation = true; - if (!ReconnectionCallback) { - const TDuration delay = UpdateReconnectionDelay(); - DEBUG_LOG("Schedule subproducer recreation through " << delay, Settings.SourceId, ""); - std::weak_ptr<TRetryingProducer> self(shared_from_this()); - ReconnectionCallback = - PQLib->GetScheduler().Schedule(delay, - this, - [self, sourceId = Settings.SourceId, logger = Logger] { - auto selfShared = self.lock(); - WRITE_LOG("Subproducer recreation callback. self is " << (selfShared ? "OK" : "nullptr") << ", stopping: " << (selfShared ? selfShared->Stopping : false), sourceId, "", TLOG_DEBUG, logger); - if (selfShared) { - selfShared->ReconnectionCallback = nullptr; - if (!selfShared->Stopping) { - selfShared->RecreateProducer(TInstant::Now() + selfShared->Settings.StartSessionTimeout); - } - } - }); - } -} - -void TRetryingProducer::ProcessStart(const NThreading::TFuture<TProducerCreateResponse>& f) noexcept { - INFO_LOG("Subproducer start response: " << f.GetValue().Response, Settings.SourceId, ""); - if (Stopping) - return; - if (NeedRecreation) - return; - if (!StartFuture.HasValue()) - return; - if (StartFuture.GetValue().Response.HasError()) { - WARN_LOG("Subproducer start error: " << f.GetValue().Response, Settings.SourceId, ""); - ScheduleRecreation(); - } else { - LastReconnectionDelay = TDuration::Zero(); - ReconnectionAttemptsDone = 0; - - // recreate on dead - DEBUG_LOG("Subscribe on subproducer death", Settings.SourceId, ""); - std::weak_ptr<TRetryingProducer> self(shared_from_this()); - PQLib->Subscribe(Producer->IsDead(), - this, - [self](const NThreading::TFuture<TError>& error) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->OnProducerDead(error.GetValue()); - } - }); - - if (!StartPromise.HasValue()) { - StartPromise.SetValue(StartFuture.GetValue()); - } - - SendData(); - } -} - -void TRetryingProducer::OnProducerDead(const TError& error) { - WARN_LOG("Subproducer is dead: " << error, Settings.SourceId, ""); - ScheduleRecreation(); -} - -void TRetryingProducer::SendData() noexcept { - if (Stopping) - return; - ui64 maxSeqNo = StartFuture.GetValue().Response.GetInit().GetMaxSeqNo(); - std::deque<NThreading::TPromise<TProducerCommitResponse>> promises; - std::deque<TWriteData> values; - ui64 prevSeqNo = 0; - for (auto& d : ResendRequests) { - Y_VERIFY(d.SeqNo == 0 || d.SeqNo > prevSeqNo); - if (d.SeqNo != 0) { - prevSeqNo = d.SeqNo; - } - if (d.SeqNo != 0 && d.SeqNo <= maxSeqNo) { - promises.push_back(Promises.front()); - Promises.pop_front(); - values.push_back(d); - } else { - DelegateWriteAndSubscribe(d.SeqNo, std::move(d.Data)); - } - } - //Requests can be not checked - it is up to client - for (auto& d : Requests) { - DelegateWriteAndSubscribe(d.SeqNo, std::move(d.Data)); - } - ResendRequests.clear(); - Requests.clear(); - for (ui32 i = 0; i < promises.size(); ++i) { - TWriteResponse res; - res.MutableAck()->SetAlreadyWritten(true); - res.MutableAck()->SetSeqNo(values[i].SeqNo); - promises[i].SetValue(TProducerCommitResponse(values[i].SeqNo, std::move(values[i].Data), std::move(res))); - } -} - -void TRetryingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept { - Y_VERIFY(data.IsEncoded()); - if (!StartFuture.Initialized()) { - TWriteResponse res; - res.MutableError()->SetDescription("producer is not ready"); - res.MutableError()->SetCode(NErrorCode::ERROR); - promise.SetValue(TProducerCommitResponse{seqNo, std::move(data), std::move(res)}); - return; - } - - Promises.push_back(promise); - - if (!StartFuture.HasValue() || NeedRecreation || Stopping || !Requests.empty() || !ResendRequests.empty()) { - Requests.emplace_back(seqNo, std::move(data)); - } else { - DelegateWriteAndSubscribe(seqNo, std::move(data)); - } -} - -void TRetryingProducer::DelegateWriteAndSubscribe(TProducerSeqNo seqNo, TData&& data) noexcept { - Y_VERIFY(InFlightRequests.size() == Futures.size()); - if (seqNo == 0) { - Futures.push_back(Producer->Write(data)); - } else { - Futures.push_back(Producer->Write(seqNo, data)); - } - InFlightRequests.emplace_back(seqNo, std::move(data)); - - std::weak_ptr<TRetryingProducer> self(shared_from_this()); - PQLib->Subscribe(Futures.back(), - this, - [self](const NThreading::TFuture<TProducerCommitResponse>&) { - auto selfShared = self.lock(); - if (selfShared) { - selfShared->ProcessFutures(); - } - }); -} - -NThreading::TFuture<TError> TRetryingProducer::IsDead() noexcept { - return IsDeadPromise.GetFuture(); -} - -void TRetryingProducer::ProcessFutures() noexcept { - NThreading::TPromise<TProducerCommitResponse> promise; - NThreading::TFuture<TProducerCommitResponse> future; - { - if (Stopping) { - return; - } - ++ToProcess; - while (ToProcess) { - if (Futures.empty() || !Futures.front().HasValue()) { - break; - } - - Y_VERIFY(InFlightRequests.size() == Futures.size()); - --ToProcess; - - const TProducerCommitResponse& response = Futures.front().GetValue(); - const TWriteData& writeData = InFlightRequests.front(); - Y_VERIFY(Promises.size() == Futures.size() + Requests.size() + ResendRequests.size()); - - if (NeedRecreation) { - ResendRequests.emplace_back(response.SeqNo, TData(writeData.Data)); - InFlightRequests.pop_front(); - Futures.pop_front(); - continue; - } - - if (Futures.front().GetValue().Response.HasError()) { - Y_VERIFY(!NeedRecreation); - ScheduleRecreation(); - WARN_LOG("Future response with error: " << response.Response, Settings.SourceId, ""); - ResendRequests.emplace_back(response.SeqNo, TData(writeData.Data)); - InFlightRequests.pop_front(); - Futures.pop_front(); - Producer = nullptr; - continue; - } - - promise = Promises.front(); - Promises.pop_front(); - future = Futures.front(); - InFlightRequests.pop_front(); - Futures.pop_front(); - promise.SetValue(future.GetValue()); - } - if (NeedRecreation && Futures.empty() && !ReconnectionCallback) { - // We need recreation, but scheduled recreation hasn't start producer because of nonempty future list. - DEBUG_LOG("Recreating subproducer after all futures were processed", Settings.SourceId, ""); - RecreateProducer(TInstant::Now() + Settings.StartSessionTimeout); - } - } -} - -void TRetryingProducer::Destroy(const TString& description) { - TError error; - error.SetDescription(description); - error.SetCode(NErrorCode::ERROR); - Destroy(error); -} - -void TRetryingProducer::SubscribeDestroyed() { - NThreading::TPromise<void> promise = ProducersDestroyed; - auto handler = [promise](const auto&) mutable { - promise.SetValue(); - }; - if (Producer) { - WaitExceptionOrAll(DestroyedPromise.GetFuture(), Producer->Destroyed()) - .Subscribe(handler); - } else { - DestroyedPromise.GetFuture() - .Subscribe(handler); - } - - DestroyPQLibRef(); -} - -void TRetryingProducer::Destroy(const TError& error) { - if (Stopping) { - return; - } - Stopping = true; - SubscribeDestroyed(); - const bool started = StartFuture.Initialized(); - Producer = nullptr; - if (started) { - Y_VERIFY(Promises.size() == ResendRequests.size() + Requests.size() + InFlightRequests.size()); - ResendRequests.insert(ResendRequests.begin(), InFlightRequests.begin(), InFlightRequests.end()); - ResendRequests.insert(ResendRequests.end(), Requests.begin(), Requests.end()); - for (auto& v : ResendRequests) { - TWriteResponse resp; - *resp.MutableError() = error; - Promises.front().SetValue(TProducerCommitResponse(v.SeqNo, std::move(v.Data), std::move(resp))); - Promises.pop_front(); - } - Y_VERIFY(Promises.empty()); - } - if (StartPromise.Initialized() && !StartPromise.HasValue()) { - TWriteResponse resp; - *resp.MutableError() = error; - StartPromise.SetValue(TProducerCreateResponse(std::move(resp))); - } - IsDeadPromise.SetValue(error); -} - -NThreading::TFuture<void> TRetryingProducer::Destroyed() noexcept { - return ProducersDestroyed.GetFuture(); -} - -void TRetryingProducer::OnStartDeadline() { - if (!StartPromise.HasValue()) { - TError error; - error.SetDescription("Start timeout."); - error.SetCode(NErrorCode::CREATE_TIMEOUT); - Destroy(error); - } -} - -void TRetryingProducer::Cancel() { - Destroy(GetCancelReason()); -} - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h deleted file mode 100644 index f1242564b90..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h +++ /dev/null @@ -1,75 +0,0 @@ -#pragma once - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include "scheduler.h" -#include "internals.h" -#include "persqueue_p.h" -#include "producer.h" -#include "iproducer_p.h" -#include <library/cpp/threading/future/future.h> -#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> -#include <deque> - -namespace NPersQueue { - -class TRetryingProducer: public IProducerImpl, public std::enable_shared_from_this<TRetryingProducer> { -public: - using IProducerImpl::Write; - // If seqno == 0, we assume it to be autodefined - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept override; - void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override { - TRetryingProducer::Write(promise, 0, std::move(data)); - } - - TRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, - TIntrusivePtr<ILogger> logger); - - NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; - - NThreading::TFuture<TError> IsDead() noexcept override; - - NThreading::TFuture<void> Destroyed() noexcept override; - - ~TRetryingProducer() noexcept; - - - void RecreateProducer(TInstant deadline) noexcept; - void ProcessStart(const NThreading::TFuture<TProducerCreateResponse>&) noexcept; - void SendData() noexcept; - void ProcessFutures() noexcept; - void DoRecreate(TInstant deadline) noexcept; - void DelegateWriteAndSubscribe(TProducerSeqNo seqNo, TData&& data) noexcept; - TDuration UpdateReconnectionDelay(); - void ScheduleRecreation(); - void Destroy(const TError& error); - void Destroy(const TString& description); // the same but with Code=ERROR - void SubscribeDestroyed(); - void OnStartDeadline(); - void OnProducerDead(const TError& error); - - void Cancel() override; - -protected: - TProducerSettings Settings; - TIntrusivePtr<ILogger> Logger; - std::shared_ptr<IProducerImpl> Producer; - std::deque<TWriteData> Requests; - std::deque<TWriteData> ResendRequests; - std::deque<TWriteData> InFlightRequests; - std::deque<NThreading::TPromise<TProducerCommitResponse>> Promises; - std::deque<NThreading::TFuture<TProducerCommitResponse>> Futures; - NThreading::TFuture<TProducerCreateResponse> StartFuture; - NThreading::TPromise<TProducerCreateResponse> StartPromise; - NThreading::TPromise<TError> IsDeadPromise; - bool NeedRecreation; - bool Stopping; - ui32 ToProcess; - TDuration LastReconnectionDelay; - unsigned ReconnectionAttemptsDone; - TIntrusivePtr<TScheduler::TCallbackHandler> ReconnectionCallback; - NThreading::TPromise<void> ProducersDestroyed = NThreading::NewPromise<void>(); -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp deleted file mode 100644 index 3fdb17d9546..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp +++ /dev/null @@ -1,355 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> - -#include <library/cpp/testing/unittest/registar.h> - -using namespace NThreading; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; - -namespace NPersQueue { - -Y_UNIT_TEST_SUITE(TRetryingProducerTest) { - TProducerSettings FakeSettings() { - TProducerSettings settings; - settings.ReconnectOnFailure = true; - settings.Server = TServerSetting{"localhost"}; - settings.Topic = "topic"; - settings.SourceId = "src"; - return settings; - } - - Y_UNIT_TEST(NotStartedProducerCanBeDestructed) { - // Test that producer doesn't hang on till shutdown - TPQLib lib; - TProducerSettings settings = FakeSettings(); - lib.CreateProducer(settings, {}, false); - } - - Y_UNIT_TEST(StartDeadlineExpires) { - if (std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED")) - return; - TPQLibSettings libSettings; - libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); - libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG); - TPQLib lib(libSettings); - THolder<IProducer> producer; - { - TProducerSettings settings = FakeSettings(); - settings.ReconnectionDelay = TDuration::MilliSeconds(1); - settings.StartSessionTimeout = TDuration::MilliSeconds(10); - producer = lib.CreateProducer(settings, {}, false); - } - const TInstant beforeStart = TInstant::Now(); - auto future = producer->Start(TDuration::MilliSeconds(100)); - UNIT_ASSERT(future.GetValueSync().Response.HasError()); - UNIT_ASSERT_EQUAL_C(future.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, - "Error: " << future.GetValueSync().Response.GetError()); - const TInstant now = TInstant::Now(); - UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(100), now); - - DestroyAndWait(producer); - } - - Y_UNIT_TEST(StartMaxAttemptsExpire) { - TPQLibSettings libSettings; - libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); - TPQLib lib(libSettings); - THolder<IProducer> producer; - { - TProducerSettings settings = FakeSettings(); - settings.ReconnectionDelay = TDuration::MilliSeconds(10); - settings.StartSessionTimeout = TDuration::MilliSeconds(10); - settings.MaxAttempts = 3; - producer = lib.CreateProducer(settings, {}, false); - } - const TInstant beforeStart = TInstant::Now(); - auto future = producer->Start(); - UNIT_ASSERT(future.GetValueSync().Response.HasError()); - const TInstant now = TInstant::Now(); - UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(30), now); - - DestroyAndWait(producer); - } - - void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { - const TProducerCommitResponse& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); - } - - void AssertWriteFailed(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { - const TProducerCommitResponse& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kError, "Msg: " << resp.Response); - } - - void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); - } - - void AssertReadContinuous(const TReadResponse::TData::TMessageBatch& batch, ui64 startSeqNo) { - for (ui32 i = 0; i < batch.MessageSize(); ++i) { - ui64 actualSeqNo = batch.GetMessage(i).GetMeta().GetSeqNo(); - UNIT_ASSERT_EQUAL_C(actualSeqNo, startSeqNo + i, "Wrong seqNo: " << actualSeqNo << ", expected: " << startSeqNo + i); - } - } - - void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response); - } - - void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response); - } - - void AssertLock(const NThreading::TFuture<TConsumerMessage>& respFuture) { - const TConsumerMessage& resp = respFuture.GetValueSync(); - UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kLock, "Msg: " << resp.Response); - } - - TProducerSettings MakeProducerSettings(const TTestServer& testServer) { - TProducerSettings producerSettings; - producerSettings.ReconnectOnFailure = true; - producerSettings.Topic = "topic1"; - producerSettings.SourceId = "123"; - producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - producerSettings.Codec = ECodec::LZOP; - producerSettings.ReconnectionDelay = TDuration::MilliSeconds(10); - return producerSettings; - } - - TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { - TConsumerSettings consumerSettings; - consumerSettings.ClientId = "user"; - consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; - consumerSettings.Topics.push_back("topic1"); - return consumerSettings; - } - - Y_UNIT_TEST(ReconnectsToServer) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - const size_t partitions = 10; - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - testServer.WaitInit("topic1"); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLib PQLib; - - auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); - producer->Start().Wait(); - TFuture<TError> isDead = producer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); - consumer->Start().Wait(); - - auto read1 = consumer->GetNextMessage(); - - // write first value - auto write1 = producer->Write(1, TString("blob1")); - AssertWriteValid(write1); - AssertReadValid(read1); - - // commit - consumer->Commit({1}); - auto commitAck = consumer->GetNextMessage(); - AssertCommited(commitAck); - - auto read2 = consumer->GetNextMessage(); - testServer.ShutdownServer(); - UNIT_ASSERT(!isDead.HasValue()); - - auto write2 = producer->Write(2, TString("blob2")); - - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - - AssertWriteValid(write2); - - AssertReadFailed(read2); - consumer->IsDead().Wait(); - - consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); - consumer->Start().Wait(); - - read2 = consumer->GetNextMessage(); - AssertReadValid(read2); - - UNIT_ASSERT(!isDead.HasValue()); - - DestroyAndWait(producer); - DestroyAndWait(consumer); - } - - static void DiesOnTooManyReconnectionAttempts(bool callWrite) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2); - testServer.WaitInit("topic1"); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLib PQLib; - - TProducerSettings settings = MakeProducerSettings(testServer); - settings.MaxAttempts = 3; - settings.ReconnectionDelay = TDuration::MilliSeconds(100); - auto producer = PQLib.CreateProducer(settings, logger, false); - producer->Start().Wait(); - TFuture<TError> isDead = producer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - // shutdown server - const TInstant beforeShutdown = TInstant::Now(); - testServer.ShutdownServer(); - - NThreading::TFuture<TProducerCommitResponse> write; - if (callWrite) { - write = producer->Write(TString("data")); - } - - isDead.Wait(); - const TInstant afterDead = TInstant::Now(); - // 3 attempts: 100ms, 200ms and 400ms - UNIT_ASSERT_C(afterDead - beforeShutdown >= TDuration::MilliSeconds(700), "real difference: " << (afterDead - beforeShutdown)); - - if (callWrite) { - AssertWriteFailed(write); - } - } - - Y_UNIT_TEST(WriteDataThatWasSentBeforeConnectToServer) { - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(false); - - testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - const size_t partitions = 10; - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - - testServer.WaitInit("topic1"); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLib PQLib; - - auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); - auto producerStarted = producer->Start(); - auto write1 = producer->Write(1, TString("a")); - producerStarted.Wait(); - TVector<TFuture<TProducerCommitResponse>> acks; - for (int i = 2; i <= 1000; ++i) { - acks.push_back(producer->Write(i, TString("b"))); - } - - AssertWriteValid(write1); - for (auto& ack: acks) { - ack.Wait(); - } - - auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); - consumer->Start().Wait(); - - ui64 readMessages = 0; - while (readMessages < 1000) { - auto read1 = consumer->GetNextMessage(); - AssertReadValid(read1); - auto& batch = read1.GetValueSync().Response.GetData().GetMessageBatch(0); - AssertReadContinuous(batch, readMessages + 1); - readMessages += batch.MessageSize(); - } - - DestroyAndWait(producer); - DestroyAndWait(consumer); - } - - Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithoutWrite) { - // Check that we reconnect even without explicit write errors - DiesOnTooManyReconnectionAttempts(false); - } - - Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithWrite) { - DiesOnTooManyReconnectionAttempts(true); - } - - Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { - return; // Test is ignored. FIX: KIKIMR-7886 - TTestServer testServer(false); - testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); - testServer.StartServer(); - - const size_t partitions = 1; - testServer.AnnoyingClient->InitRoot(); - testServer.AnnoyingClient->InitDCs(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); - testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); - testServer.AnnoyingClient->InitSourceIds(); - - testServer.WaitInit("topic1"); - - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLibSettings pqLibSettings; - pqLibSettings.DefaultLogger = logger; - THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); - - auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, false); - UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); - TFuture<TError> isDead = producer->IsDead(); - UNIT_ASSERT(!isDead.HasValue()); - - testServer.ShutdownServer(); - UNIT_ASSERT(!isDead.HasValue()); - - auto write1 = producer->Write(1, TString("blob1")); - auto write2 = producer->Write(2, TString("blob2")); - - UNIT_ASSERT(!write1.HasValue()); - UNIT_ASSERT(!write2.HasValue()); - - PQLib = nullptr; - - UNIT_ASSERT(write1.HasValue()); - UNIT_ASSERT(write2.HasValue()); - - UNIT_ASSERT(write1.GetValue().Response.HasError()); - UNIT_ASSERT(write2.GetValue().Response.HasError()); - - auto write3 = producer->Write(3, TString("blob3")); - UNIT_ASSERT(write3.HasValue()); - UNIT_ASSERT(write3.GetValue().Response.HasError()); - } - - Y_UNIT_TEST(CancelsStartAfterPQLibDeath) { - TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - TPQLibSettings pqLibSettings; - pqLibSettings.DefaultLogger = logger; - THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); - - auto producer = PQLib->CreateProducer(FakeSettings(), logger, false); - auto start = producer->Start(); - UNIT_ASSERT(!start.HasValue()); - - PQLib = nullptr; - - UNIT_ASSERT(start.HasValue()); - UNIT_ASSERT(start.GetValue().Response.HasError()); - - auto dead = producer->IsDead(); - UNIT_ASSERT(dead.HasValue()); - dead.GetValueSync(); - - auto write = producer->Write(1, TString("blob1")); - UNIT_ASSERT(write.HasValue()); - UNIT_ASSERT(write.GetValueSync().Response.HasError()); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp deleted file mode 100644 index ba6c4ebb405..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp +++ /dev/null @@ -1,96 +0,0 @@ -#include "persqueue_p.h" -#include "scheduler.h" - -#include <util/system/thread.h> - -namespace NPersQueue { -void TScheduler::TCallbackHandler::Execute() { - auto guard = Guard(Lock); - if (Callback) { - Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(std::move(Callback))); - Callback = nullptr; - } -} - -void TScheduler::TCallbackHandler::TryCancel() { - auto guard = Guard(Lock); - if (Callback) { - Callback = nullptr; - } -} - -bool TScheduler::TCallbackHandlersCompare::operator()(const TIntrusivePtr<TScheduler::TCallbackHandler>& h1, const TIntrusivePtr<TScheduler::TCallbackHandler>& h2) const { - return h1->Time > h2->Time; -} - -TScheduler::TScheduler(TPQLibPrivate* pqLib) - : Shutdown(false) - , PQLib(pqLib) -{ - Thread = SystemThreadFactory()->Run([this] { - this->SchedulerThread(); - }); -} - -TScheduler::~TScheduler() { - ShutdownAndWait(); -} - -void TScheduler::AddToSchedule(TIntrusivePtr<TCallbackHandler> handler) { - { - auto guard = Guard(Lock); - Callbacks.push(std::move(handler)); - } - Event.Signal(); -} - -void TScheduler::ShutdownAndWait() { - AtomicSet(Shutdown, true); - Event.Signal(); - Thread->Join(); -} - -void TScheduler::SchedulerThread() { - TThread::SetCurrentThreadName("pqlib_scheduler"); - while (!AtomicGet(Shutdown)) { - TInstant deadline = TInstant::Max(); - std::vector<TIntrusivePtr<TCallbackHandler>> callbacks; - { - // define next deadline and get expired callbacks - auto guard = Guard(Lock); - const TInstant now = TInstant::Now(); - while (!Callbacks.empty()) { - const auto& top = Callbacks.top(); - if (top->Time <= now) { - callbacks.push_back(top); - Callbacks.pop(); - } else { - deadline = top->Time; - break; - } - } - } - - // execute callbacks - bool shutdown = false; - for (auto& callback : callbacks) { - if (shutdown) { - callback->TryCancel(); - } else { - callback->Execute(); - } - shutdown = shutdown || AtomicGet(Shutdown); - } - - if (!shutdown) { - Event.WaitD(deadline); - } - } - - // cancel all callbacks and clear data - while (!Callbacks.empty()) { - Callbacks.top()->TryCancel(); - Callbacks.pop(); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h deleted file mode 100644 index c90ab87795f..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h +++ /dev/null @@ -1,88 +0,0 @@ -#pragma once -#include <util/datetime/base.h> -#include <util/generic/ptr.h> -#include <util/system/event.h> -#include <util/system/spinlock.h> -#include <util/thread/factory.h> - -#include <functional> -#include <queue> -#include <utility> - -namespace NPersQueue { - -class TPQLibPrivate; - -class TScheduler { - struct TCallbackHandlersCompare; -public: - using TCallback = std::function<void()>; - - class TCallbackHandler: public TAtomicRefCount<TCallbackHandler> { - friend class TScheduler; - friend struct TCallbackHandlersCompare; - - template <class TFunc> - TCallbackHandler(TInstant time, TFunc&& func, TPQLibPrivate* pqLib, const void* queueTag) - : Time(time) - , Callback(std::forward<TFunc>(func)) - , PQLib(pqLib) - , QueueTag(queueTag) - { - } - - void Execute(); - - public: - // Cancels execution of callback. - // If callback is already canceled or executes (executed), does nothing. - // Posteffect: callback is guaranteed to be destroyed after this call. - void TryCancel(); - - private: - TInstant Time; - TCallback Callback; - TAdaptiveLock Lock; - TPQLibPrivate* PQLib; - const void* QueueTag; - }; - -public: - // Starts a scheduler thread. - explicit TScheduler(TPQLibPrivate* pqLib); - - // Stops a scheduler thread. - ~TScheduler(); - - // Schedules a new callback to be executed - template <class TFunc> - TIntrusivePtr<TCallbackHandler> Schedule(TInstant time, const void* queueTag, TFunc&& func) { - TIntrusivePtr<TCallbackHandler> handler(new TCallbackHandler(time, std::forward<TFunc>(func), PQLib, queueTag)); - AddToSchedule(handler); - return handler; - } - - template <class TFunc> - TIntrusivePtr<TCallbackHandler> Schedule(TDuration delta, const void* queueTag, TFunc&& func) { - return Schedule(TInstant::Now() + delta, queueTag, std::forward<TFunc>(func)); - } - -private: - void AddToSchedule(TIntrusivePtr<TCallbackHandler> handler); - void ShutdownAndWait(); - void SchedulerThread(); - -private: - struct TCallbackHandlersCompare { - bool operator()(const TIntrusivePtr<TCallbackHandler>& h1, const TIntrusivePtr<TCallbackHandler>& h2) const; - }; - - using TCallbackQueue = std::priority_queue<TIntrusivePtr<TCallbackHandler>, std::vector<TIntrusivePtr<TCallbackHandler>>, TCallbackHandlersCompare>; - TAdaptiveLock Lock; - TAutoEvent Event; - TCallbackQueue Callbacks; - TAtomic Shutdown; - THolder<IThreadFactory::IThread> Thread; - TPQLibPrivate* PQLib; -}; -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp deleted file mode 100644 index 83ce25c6b49..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "scheduler.h" -#include "persqueue_p.h" - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/system/event.h> - -namespace NPersQueue { -const void* Tag = (const void*)42; -Y_UNIT_TEST_SUITE(TSchedulerTest) { - Y_UNIT_TEST(EmptySchedulerCanDestruct) { - // Test that scheduler doesn't hang on till shutdown - TScheduler scheduler(nullptr); - } - - Y_UNIT_TEST(ExecutesInProperOrder) { - TSystemEvent ev; - bool callback1Executed = false; - TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({})); - TScheduler scheduler(lib.Get()); - scheduler.Schedule(TDuration::MilliSeconds(100), Tag, [&callback1Executed, &ev] { - UNIT_ASSERT(callback1Executed); - ev.Signal(); - }); - scheduler.Schedule(TDuration::MilliSeconds(50), Tag, [&callback1Executed] { - callback1Executed = true; - }); - ev.Wait(); - } - - Y_UNIT_TEST(CancelsAndClearsData) { - TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({})); - TScheduler scheduler(lib.Get()); - std::shared_ptr<TString> ptr(new TString()); - bool callbackExecuted = false; - auto h = scheduler.Schedule(TDuration::Seconds(50), Tag, [&callbackExecuted, ptr] { - callbackExecuted = true; - }); - UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 2); - h->TryCancel(); - UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1); - UNIT_ASSERT(!callbackExecuted); - Sleep(TDuration::MilliSeconds(51)); - UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1); - UNIT_ASSERT(!callbackExecuted); - } - - Y_UNIT_TEST(ExitsThreadImmediately) { - TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({})); - std::shared_ptr<TString> ptr(new TString()); - bool callback1Executed = false; - bool callback2Executed = false; - TSystemEvent ev, ev2; - auto now = TInstant::Now(); - lib->GetScheduler().Schedule(TDuration::Seconds(500), Tag, [&callback1Executed, ptr] { - callback1Executed = true; - }); - lib->GetScheduler().Schedule(now, Tag, [&callback2Executed, &ev, &ev2, ptr] { - callback2Executed = true; - ev2.Wait(); - ev.Signal(); - }); - UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 3); - ev2.Signal(); - - // kill scheduler - ev.Wait(); - lib.Reset(); - - UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1); - UNIT_ASSERT(!callback1Executed); - UNIT_ASSERT(callback2Executed); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp deleted file mode 100644 index 4fc6a1c7b44..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> - -#include <library/cpp/streams/lzop/lzop.h> -#include <library/cpp/streams/zstd/zstd.h> - -#include <util/generic/store_policy.h> -#include <util/generic/utility.h> -#include <util/stream/str.h> -#include <util/stream/zlib.h> - -namespace NPersQueue { - -const TString& TServerSetting::GetRootDatabase() { - static const TString RootDatabase = "/Root"; - return RootDatabase; -} - -class TZLibToStringCompressor: private TEmbedPolicy<TStringOutput>, public TZLibCompress { -public: - TZLibToStringCompressor(TString& dst, ZLib::StreamType type, size_t quality) - : TEmbedPolicy<TStringOutput>(dst) - , TZLibCompress(TEmbedPolicy::Ptr(), type, quality) - { - } -}; - -class TLzopToStringCompressor: private TEmbedPolicy<TStringOutput>, public TLzopCompress { -public: - TLzopToStringCompressor(TString& dst) - : TEmbedPolicy<TStringOutput>(dst) - , TLzopCompress(TEmbedPolicy::Ptr()) - { - } -}; - -class TZstdToStringCompressor: private TEmbedPolicy<TStringOutput>, public TZstdCompress { -public: - TZstdToStringCompressor(TString& dst, int quality) - : TEmbedPolicy<TStringOutput>(dst) - , TZstdCompress(TEmbedPolicy::Ptr(), quality) - { - } -}; - - -TData TData::Encode(TData source, ECodec defaultCodec, int quality) { - Y_VERIFY(!source.Empty()); - TData data = std::move(source); - if (data.IsEncoded()) { - return data; - } - Y_VERIFY(defaultCodec != ECodec::RAW && defaultCodec != ECodec::DEFAULT); - if (data.Codec == ECodec::DEFAULT) { - data.Codec = defaultCodec; - } - THolder<IOutputStream> coder = CreateCoder(data.Codec, data, quality); - coder->Write(data.SourceData); - coder->Finish(); // &data.EncodedData may be already invalid on coder destruction - return data; -} - -TData TData::MakeRawIfNotEncoded(TData source) { - Y_VERIFY(!source.Empty()); - TData data = std::move(source); - if (!data.IsEncoded()) { - data.EncodedData = data.SourceData; - data.Codec = ECodec::RAW; - } - return data; -} - -THolder<IOutputStream> TData::CreateCoder(ECodec codec, TData& result, int quality) { - switch (codec) { - case ECodec::GZIP: - return MakeHolder<TZLibToStringCompressor>(result.EncodedData, ZLib::GZip, quality >= 0 ? quality : 6); - case ECodec::LZOP: - return MakeHolder<TLzopToStringCompressor>(result.EncodedData); - case ECodec::ZSTD: - return MakeHolder<TZstdToStringCompressor>(result.EncodedData, quality); - default: - Y_FAIL("NOT IMPLEMENTED CODEC TYPE"); - } -} - - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp deleted file mode 100644 index 06c4b284544..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <util/string/builder.h> - -namespace NPersQueue { -Y_UNIT_TEST_SUITE(TDataTest) { - ECodec Codecs[] = { - ECodec::LZOP, - ECodec::GZIP, - }; - - TString DebugString(const TData& data) { - return TStringBuilder() << "{ Ts: " << data.GetTimestamp().GetValue() << ", SrcData: \"" << data.GetSourceData() - << "\", Encoded: \"" << (data.IsEncoded() ? data.GetEncodedData() : TString()) - << "\", Codec: " << data.GetCodecType() << " }"; - } - - Y_UNIT_TEST(RawDataIsEncoded) { - auto now = TInstant::Now(); - TData data = TData::Raw("trololo", now); - TData otherRawData = TData("trololo", ECodec::RAW, now); - UNIT_ASSERT_C(data.IsEncoded(), "data: " << DebugString(data)); - UNIT_ASSERT_C(!data.Empty(), "data: " << DebugString(data)); - UNIT_ASSERT_EQUAL_C(data.GetCodecType(), ECodec::RAW, "data: " << DebugString(data)); - UNIT_ASSERT_EQUAL_C(data, otherRawData, "data: " << DebugString(data) << ", other: " << DebugString(otherRawData)); - } - - Y_UNIT_TEST(EncodedDataIsEncoded) { - for (ECodec codec : Codecs) { - TData data = TData::Encoded("trololo", codec); - UNIT_ASSERT(data.IsEncoded()); - UNIT_ASSERT(!data.Empty()); - UNIT_ASSERT_EQUAL(data.GetCodecType(), codec); - } - } - - Y_UNIT_TEST(ModifiesState) { - for (ECodec codec : Codecs) { - for (ECodec defaultCodec : Codecs) { - TData data("trololo", codec); - UNIT_ASSERT(!data.IsEncoded()); - UNIT_ASSERT(!data.Empty()); - - for (size_t i = 0; i < 2; ++i) { - data = TData::Encode(std::move(data), defaultCodec, 2); // encode twice is OK - UNIT_ASSERT(data.IsEncoded()); - UNIT_ASSERT(!data.Empty()); - UNIT_ASSERT(!data.GetEncodedData().empty()); - UNIT_ASSERT_EQUAL(data.GetCodecType(), codec); - } - } - } - } - - Y_UNIT_TEST(HandlesDefaultCodec) { - for (ECodec defaultCodec : Codecs) { - TData data = TString("trololo"); - UNIT_ASSERT(!data.IsEncoded()); - UNIT_ASSERT(!data.Empty()); - - data = TData::Encode(data, defaultCodec, -1); - UNIT_ASSERT(data.IsEncoded()); - UNIT_ASSERT(!data.Empty()); - UNIT_ASSERT_STRINGS_EQUAL(data.GetSourceData(), "trololo"); - UNIT_ASSERT(!data.GetEncodedData().empty()); - UNIT_ASSERT_EQUAL(data.GetCodecType(), defaultCodec); - } - } - - Y_UNIT_TEST(MakesRaw) { - TData data = TString("trololo"); - UNIT_ASSERT(!data.IsEncoded()); - data = TData::MakeRawIfNotEncoded(data); - UNIT_ASSERT(data.IsEncoded()); - UNIT_ASSERT_EQUAL(data.GetCodecType(), ECodec::RAW); - } - - Y_UNIT_TEST(DoesNotMakeRaw) { - TData data = TData::Encoded("trololo", ECodec::GZIP); - UNIT_ASSERT(data.IsEncoded()); - data = TData::MakeRawIfNotEncoded(data); - UNIT_ASSERT(data.IsEncoded()); - UNIT_ASSERT_EQUAL(data.GetCodecType(), ECodec::GZIP); - } -} -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp deleted file mode 100644 index a7d65ddc88b..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "validate_grpc_metadata.h" -#include <contrib/libs/grpc/include/grpc/grpc.h> -#include <util/generic/string.h> -#include <util/string/escape.h> -#include <util/string/builder.h> - -namespace NGrpc { - bool ValidateHeaderIsLegal(const TString& key, const TString& value, TString& error) { - error.clear(); - grpc_slice keySlice = grpc_slice_from_static_buffer(key.c_str(), key.size()); - int ok = grpc_header_key_is_legal(keySlice); - if (!ok) { - error = TStringBuilder() << "gRPC metadata header key is illegal: \"" << EscapeC(key) << "\""; - return false; - } - grpc_slice valueSlice = grpc_slice_from_static_buffer(value.c_str(), value.size()); - ok = grpc_is_binary_header(keySlice) || grpc_header_nonbin_value_is_legal(valueSlice); - if (!ok) { - error = TStringBuilder() << "gRPC metadata header value with key \"" << key << "\" is illegal: \"" << EscapeC(value) << "\""; - return false; - } - return true; - } -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h deleted file mode 100644 index 43fd82cef02..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once -#include <util/generic/fwd.h> - -namespace NGrpc { - // Validates gRPC metadata header key and value. Returns 'true' if validations fails, otherwise returns 'false' and sets 'error' value. - // For more information see 'Custom-Metadata' section in gRPC over HTTP2 (https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md). - // Note that in case of authentication data validation 'error' may contain sensitive information. - bool ValidateHeaderIsLegal(const TString& key, const TString& value, TString& error); - - // Creates hexadecimal representation of 's' in format '{0x01, 0x02, 0x03}' - TString ToHexString(const TString& s); -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp deleted file mode 100644 index bbda57f5387..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "validate_grpc_metadata.h" -#include <library/cpp/testing/unittest/registar.h> - -using namespace NGrpc; - -Y_UNIT_TEST_SUITE(NValidateGrpcMetadata) { - Y_UNIT_TEST(ValidateHeaderIsLegal) { - TString error = "error"; - UNIT_ASSERT_C(ValidateHeaderIsLegal("key", "value", error), error); - // Assert 'error' is cleared - UNIT_ASSERT_C(error.empty(), error); - - // Valid character values upper and lower bounds - UNIT_ASSERT_C(ValidateHeaderIsLegal("\x30\x39\x61\x7A_-.", "\x20\x7E", error), error); - UNIT_ASSERT_C(error.empty(), error); - - TString null = " "; - null[0] = '\0'; - UNIT_ASSERT(!ValidateHeaderIsLegal("key", null, error)); - UNIT_ASSERT_C(error.Contains("\\0"), error); - Cerr << "Error is '" << error << "'" << Endl; - - // Simple escape sequences - UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x0A\t", error)); - UNIT_ASSERT_C(error.Contains("\\n\\t"), error); - Cerr << "Error is '" << error << "'" << Endl; - - UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x1F", error)); - UNIT_ASSERT_C(error.Contains("\\x1F"), error); - Cerr << "Error is '" << error << "'" << Endl; - - UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x7F", error)); - UNIT_ASSERT_C(error.Contains("\\x7F"), error); - Cerr << "Error is '" << error << "'" << Endl; - - // Octal character - UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\177", error)); - UNIT_ASSERT_C(error.Contains("\\x7F"), error); - Cerr << "Error is '" << error << "'" << Endl; - - // Invalid header key - UNIT_ASSERT(!ValidateHeaderIsLegal("key\n", "value", error)); - UNIT_ASSERT_C(error.Contains("\\n"), error); - Cerr << "Error is '" << error << "'" << Endl; - } -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp deleted file mode 100644 index b679eda9f4a..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp +++ /dev/null @@ -1,561 +0,0 @@ -#include "ydb_sdk_consumer.h" -#include "persqueue_p.h" - -#include <ydb/library/persqueue/topic_parser_public/topic_parser.h> - -#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h> -#include <util/generic/is_in.h> -#include <util/string/builder.h> - -namespace NPersQueue { - -static TError MakeError(const TString& description, NErrorCode::EErrorCode code) { - TError error; - error.SetDescription(description); - error.SetCode(code); - return error; -} - -static TString MakeLegacyTopicName(const NYdb::NPersQueue::TPartitionStream::TPtr& partitionStream) { - // rt3.man--account--topic - return BuildFullTopicName(partitionStream->GetTopicPath(), partitionStream->GetCluster()); -} - -static NErrorCode::EErrorCode ToErrorCode(NYdb::EStatus status) { - switch (status) { - case NYdb::EStatus::STATUS_UNDEFINED: - return NErrorCode::ERROR; - case NYdb::EStatus::SUCCESS: - return NErrorCode::OK; - case NYdb::EStatus::BAD_REQUEST: - return NErrorCode::BAD_REQUEST; - case NYdb::EStatus::UNAUTHORIZED: - return NErrorCode::ACCESS_DENIED; - case NYdb::EStatus::INTERNAL_ERROR: - return NErrorCode::ERROR; - case NYdb::EStatus::ABORTED: - [[fallthrough]]; - case NYdb::EStatus::UNAVAILABLE: - return NErrorCode::ERROR; - case NYdb::EStatus::OVERLOADED: - return NErrorCode::OVERLOAD; - case NYdb::EStatus::SCHEME_ERROR: - return NErrorCode::UNKNOWN_TOPIC; - case NYdb::EStatus::GENERIC_ERROR: - [[fallthrough]]; - case NYdb::EStatus::TIMEOUT: - [[fallthrough]]; - case NYdb::EStatus::BAD_SESSION: - [[fallthrough]]; - case NYdb::EStatus::PRECONDITION_FAILED: - [[fallthrough]]; - case NYdb::EStatus::ALREADY_EXISTS: - [[fallthrough]]; - case NYdb::EStatus::NOT_FOUND: - [[fallthrough]]; - case NYdb::EStatus::SESSION_EXPIRED: - [[fallthrough]]; - case NYdb::EStatus::CANCELLED: - [[fallthrough]]; - case NYdb::EStatus::UNDETERMINED: - [[fallthrough]]; - case NYdb::EStatus::UNSUPPORTED: - [[fallthrough]]; - case NYdb::EStatus::SESSION_BUSY: - [[fallthrough]]; - case NYdb::EStatus::TRANSPORT_UNAVAILABLE: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_RESOURCE_EXHAUSTED: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_DEADLINE_EXCEEDED: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_INTERNAL_ERROR: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_CANCELLED: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_UNAUTHENTICATED: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_CALL_UNIMPLEMENTED: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_OUT_OF_RANGE: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_DISCOVERY_FAILED: - [[fallthrough]]; - case NYdb::EStatus::CLIENT_LIMITS_REACHED: - return NErrorCode::ERROR; - } -} - -TYdbSdkCompatibilityConsumer::TYdbSdkCompatibilityConsumer(const TConsumerSettings& settings, - std::shared_ptr<void> destroyEventRef, - TIntrusivePtr<TPQLibPrivate> pqLib, - TIntrusivePtr<ILogger> logger, - NYdb::NPersQueue::TPersQueueClient& client) - : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) - , Settings(settings) - , Logger(std::move(logger)) - , Client(client) -{ -} - -TYdbSdkCompatibilityConsumer::~TYdbSdkCompatibilityConsumer() { - Destroy("Destructor called"); -} - -void TYdbSdkCompatibilityConsumer::Init() { - YdbSdkSettings = MakeReadSessionSettings(); // weak_from_this() is used here. -} - -static const THashSet<TString> FederationClusters = { - "iva", - "man", - "myt", - "sas", - "vla", -}; - -static TString FindClusterFromEndpoint(const TString& endpoint) { - size_t pos = endpoint.find(".logbroker.yandex.net"); - if (pos == TString::npos) { - pos = endpoint.find(".logbroker-prestable.yandex.net"); - if (pos == TString::npos) { - return {}; // something strange or cross dc cloud cluster. - } - } - TString prefix = endpoint.substr(0, pos); - if (IsIn(FederationClusters, prefix)) { - return prefix; - } - return {}; // no cluster in cross dc federation. -} - -NYdb::NPersQueue::TReadSessionSettings TYdbSdkCompatibilityConsumer::MakeReadSessionSettings() { - NYdb::NPersQueue::TReadSessionSettings settings; - for (const TString& topic : Settings.Topics) { - settings.AppendTopics(topic); - } - settings.ConsumerName(Settings.ClientId); - settings.MaxMemoryUsageBytes(Settings.MaxMemoryUsage); - if (Settings.MaxTimeLagMs) { - settings.MaxTimeLag(TDuration::MilliSeconds(Settings.MaxTimeLagMs)); - } - if (Settings.ReadTimestampMs) { - settings.StartingMessageTimestamp(TInstant::MilliSeconds(Settings.ReadTimestampMs)); - } - if (Settings.PartitionGroups.size()) { - Y_ENSURE(settings.Topics_.size() == 1); - for (ui64 group : Settings.PartitionGroups) { - settings.Topics_[0].AppendPartitionGroupIds(group); - } - } - if (Settings.ReconnectOnFailure || Settings.ReadFromAllClusterSources) { // ReadFromAllClusterSources implies ReconnectOnFailure and MaxAttempts == inf. - size_t maxRetries = Settings.MaxAttempts; - if (Settings.ReadFromAllClusterSources) { - // Compatibility. - maxRetries = std::numeric_limits<size_t>::max(); - } - - if (Settings.UseV2RetryPolicyInCompatMode) { - settings.RetryPolicy( - NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy( - Settings.ReconnectionDelay, - Settings.ReconnectionDelay, - Settings.MaxReconnectionDelay, - maxRetries, - TDuration::Max(), - 2.0, - NYdb::NPersQueue::GetRetryErrorClassV2 - )); - } else { - settings.RetryPolicy( - NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy( - Settings.ReconnectionDelay, - Settings.ReconnectionDelay, - Settings.MaxReconnectionDelay, - maxRetries - )); - } - } else { - settings.RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()); - } - if (Settings.ReadFromAllClusterSources) { - settings.ReadAll(); - } else { - if (TString cluster = FindClusterFromEndpoint(Settings.Server.Address)) { - if (Settings.ReadMirroredPartitions) { - settings.ReadMirrored(cluster); - } else { - settings.ReadOriginal({ cluster }); - } - } else { - settings.ReadAll(); - } - } - if (Settings.DisableCDS) { - settings.DisableClusterDiscovery(true); - } - - settings.EventHandlers_.HandlersExecutor(NYdb::NPersQueue::CreateThreadPoolExecutorAdapter(PQLib->GetQueuePool().GetQueuePtr(this))); - { - auto weakThis = weak_from_this(); - settings.EventHandlers_.SessionClosedHandler( - [weakThis](const NYdb::NPersQueue::TSessionClosedEvent& event) { - if (auto sharedThis = weakThis.lock()) { - const TString description = event.GetIssues().ToString(); - sharedThis->Destroy(description, ToErrorCode(event.GetStatus())); - } - } - ); - } - - return settings; -} - -NThreading::TFuture<TConsumerCreateResponse> TYdbSdkCompatibilityConsumer::Start(TInstant) noexcept { - ReadSession = Client.CreateReadSession(YdbSdkSettings); - SessionId = ReadSession->GetSessionId(); - DEBUG_LOG("Create read session", "", SessionId); - SubscribeToNextEvent(); - TReadResponse resp; - resp.MutableInit(); - return NThreading::MakeFuture<TConsumerCreateResponse>(TConsumerCreateResponse(std::move(resp))); -} - -NThreading::TFuture<TError> TYdbSdkCompatibilityConsumer::IsDead() noexcept { - return DeadPromise.GetFuture(); -} - -void TYdbSdkCompatibilityConsumer::Destroy(const TError& description) { - if (DeadPromise.HasValue()) { - return; - } - - WARN_LOG("Destroying consumer: " << description, "", SessionId); - - while (!Requests.empty()) { - NPersQueue::TReadResponse resp; - *resp.MutableError() = description; - Requests.front().SetValue(TConsumerMessage(std::move(resp))); - Requests.pop(); - } - - if (ReadSession) { - ReadSession->Close(TDuration::Zero()); - ReadSession = nullptr; - } - - DeadPromise.SetValue(description); - - DestroyPQLibRef(); -} - -void TYdbSdkCompatibilityConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) { - Destroy(MakeError(description, code)); -} - -void TYdbSdkCompatibilityConsumer::Cancel() { - Destroy(GetCancelReason()); -} - -void TYdbSdkCompatibilityConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { - if (DeadPromise.HasValue()) { - NPersQueue::TReadResponse resp; - *resp.MutableError() = DeadPromise.GetFuture().GetValue(); - promise.SetValue(TConsumerMessage(std::move(resp))); - return; - } - Requests.push(promise); - AnswerToRequests(); - SubscribeToNextEvent(); -} - -void TYdbSdkCompatibilityConsumer::Commit(const TVector<ui64>& cookies) noexcept { - NPersQueue::TReadResponse resp; - for (ui64 cookie : cookies) { - if (cookie >= NextCookie) { - Destroy(TStringBuilder() << "Wrong cookie " << cookie, NErrorCode::WRONG_COOKIE); - return; - } - auto offsetsIt = CookieToOffsets.find(cookie); - if (offsetsIt != CookieToOffsets.end()) { - offsetsIt->second.first.Commit(); - CookiesRequestedToCommit[offsetsIt->second.second].emplace(cookie); - CookieToOffsets.erase(offsetsIt); - } else { - resp.MutableCommit()->AddCookie(cookie); - } - } - if (resp.HasCommit()) { - AddResponse(std::move(resp)); - } -} - -void TYdbSdkCompatibilityConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { - Y_UNUSED(topic); - Y_UNUSED(partition); - auto partitionStreamIt = CurrentPartitionStreams.find(generation); - if (partitionStreamIt != CurrentPartitionStreams.end()) { - partitionStreamIt->second->RequestStatus(); - } -} - -void TYdbSdkCompatibilityConsumer::AnswerToRequests() { - TVector<NYdb::NPersQueue::TReadSessionEvent::TEvent> events; - do { - events = ReadSession->GetEvents(); - for (NYdb::NPersQueue::TReadSessionEvent::TEvent& event : events) { - if (ReadSession) { - std::visit([this](auto&& ev) { return HandleEvent(std::move(ev)); }, event); - } - } - } while (!events.empty() && ReadSession); - - while (!Requests.empty() && !Responses.empty()) { - Requests.front().SetValue(std::move(Responses.front())); - Requests.pop(); - Responses.pop(); - } -} - -void TYdbSdkCompatibilityConsumer::SubscribeToNextEvent() { - if (!SubscribedToNextEvent && !Requests.empty() && ReadSession) { - SubscribedToNextEvent = true; - auto weakThis = weak_from_this(); - auto future = ReadSession->WaitEvent(); - Cerr << "SUBSCRIBING " << ReadSession->GetSessionId() << " future" << future.StateId()->Value() << "\n"; - PQLib->Subscribe(future, this, [weakThis](const NThreading::TFuture<void>&) { - if (auto sharedThis = weakThis.lock()) { - sharedThis->OnReadSessionEvent(); - } - }); - } -} - -void TYdbSdkCompatibilityConsumer::OnReadSessionEvent() { - if (DeadPromise.HasValue()) { - return; - } - - SubscribedToNextEvent = false; - AnswerToRequests(); - SubscribeToNextEvent(); -} - -void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent&& event) { - if (Settings.UseLockSession) { - auto& offsetSet = PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()]; - // Messages could contain holes in offset, but later commit ack will tell us right border. - // So we can easily insert the whole interval with holes included. - // It will be removed from set by specifying proper right border. - offsetSet.InsertInterval(event.GetMessages().front().GetOffset(), event.GetMessages().back().GetOffset() + 1); - } - const ui64 cookie = NextCookie++; - const auto& partitionStream = event.GetPartitionStream(); - auto& offsets = CookieToOffsets[cookie]; - offsets.first.Add(event); - offsets.second = partitionStream->GetPartitionStreamId(); - PartitionStreamToCookies[partitionStream->GetPartitionStreamId()].emplace(cookie); - NPersQueue::TReadResponse resp; - auto& dataResp = *resp.MutableData(); - dataResp.SetCookie(cookie); - auto& batchResp = *dataResp.AddMessageBatch(); - batchResp.SetTopic(MakeLegacyTopicName(partitionStream)); - batchResp.SetPartition(partitionStream->GetPartitionId()); - ui64 maxOffset = 0; - for (auto&& msg : event.GetMessages()) { - auto& msgResp = *batchResp.AddMessage(); - msgResp.SetOffset(msg.GetOffset()); - maxOffset = Max(maxOffset, msg.GetOffset()); - msgResp.SetData(msg.GetData()); - auto& metaResp = *msgResp.MutableMeta(); - metaResp.SetSourceId(msg.GetMessageGroupId()); - metaResp.SetSeqNo(msg.GetSeqNo()); - metaResp.SetCreateTimeMs(msg.GetCreateTime().MilliSeconds()); - metaResp.SetWriteTimeMs(msg.GetWriteTime().MilliSeconds()); - metaResp.SetIp(msg.GetIp()); - for (auto&& [k, v] : msg.GetMeta()->Fields) { - auto& kvResp = *metaResp.MutableExtraFields()->AddItems(); - kvResp.SetKey(k); - kvResp.SetValue(v); - } - } - MaxOffsetToCookie[std::make_pair(partitionStream->GetPartitionStreamId(), maxOffset)] = cookie; - AddResponse(std::move(resp)); -} -void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&& event) { - const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); - if (Settings.UseLockSession) { - auto& offsetSet = PartitionStreamToUncommittedOffsets[partitionStreamId]; - if (offsetSet.EraseInterval(0, event.GetCommittedOffset())) { // Remove some offsets. - if (offsetSet.Empty()) { // No offsets left. - auto unconfirmedDestroyIt = UnconfirmedDestroys.find(partitionStreamId); - if (unconfirmedDestroyIt != UnconfirmedDestroys.end()) { - // Confirm and forget about this partition stream. - unconfirmedDestroyIt->second.Confirm(); - UnconfirmedDestroys.erase(unconfirmedDestroyIt); - PartitionStreamToUncommittedOffsets.erase(partitionStreamId); - } - } - } - } - const auto offsetPair = std::make_pair(partitionStreamId, event.GetCommittedOffset() - 1); - auto cookieIt = MaxOffsetToCookie.lower_bound(offsetPair); - std::vector<ui64> cookies; - auto end = cookieIt; - auto begin = end; - if (cookieIt != MaxOffsetToCookie.end() && cookieIt->first == offsetPair) { - cookies.push_back(cookieIt->second); - ++end; - } - while (cookieIt != MaxOffsetToCookie.begin()) { - --cookieIt; - if (cookieIt->first.first == partitionStreamId) { - cookies.push_back(cookieIt->second); - begin = cookieIt; - } else { - break; - } - } - if (begin != end) { - MaxOffsetToCookie.erase(begin, end); - } - - NPersQueue::TReadResponse resp; - auto& respCommit = *resp.MutableCommit(); - auto& partitionStreamCookies = PartitionStreamToCookies[partitionStreamId]; - auto& committedCookies = CookiesRequestedToCommit[partitionStreamId]; - for (auto committedCookieIt = cookies.rbegin(), committedCookieEnd = cookies.rend(); committedCookieIt != committedCookieEnd; ++committedCookieIt) { - const ui64 cookie = *committedCookieIt; - respCommit.AddCookie(cookie); - partitionStreamCookies.erase(cookie); - committedCookies.erase(cookie); - } - AddResponse(std::move(resp)); -} - -void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent&& event) { - if (Settings.UseLockSession) { - Y_VERIFY(PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()].Empty()); - - NPersQueue::TReadResponse resp; - auto& lockResp = *resp.MutableLock(); - const auto& partitionStream = event.GetPartitionStream(); - lockResp.SetTopic(MakeLegacyTopicName(partitionStream)); - lockResp.SetPartition(partitionStream->GetPartitionId()); - lockResp.SetReadOffset(event.GetCommittedOffset()); - lockResp.SetEndOffset(event.GetEndOffset()); - lockResp.SetGeneration(partitionStream->GetPartitionStreamId()); - NThreading::TPromise<TLockInfo> confirmPromise = NThreading::NewPromise<TLockInfo>(); - confirmPromise.GetFuture().Subscribe([event = std::move(event)](const NThreading::TFuture<TLockInfo>& infoFuture) mutable { - const TLockInfo& info = infoFuture.GetValue(); - event.Confirm(info.ReadOffset, info.CommitOffset); - }); // It doesn't matter in what thread this callback will be called. - AddResponse(std::move(resp), std::move(confirmPromise)); - } else { - event.Confirm(); - } -} - -void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent&& event) { - const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); - CurrentPartitionStreams[partitionStreamId] = event.GetPartitionStream(); - if (Settings.UseLockSession) { - const auto partitionStream = event.GetPartitionStream(); - - Y_VERIFY(UnconfirmedDestroys.find(partitionStreamId) == UnconfirmedDestroys.end()); - if (PartitionStreamToUncommittedOffsets[partitionStreamId].Empty() || Settings.BalanceRightNow) { - PartitionStreamToUncommittedOffsets.erase(partitionStreamId); - event.Confirm(); - } else { - UnconfirmedDestroys.emplace(partitionStreamId, std::move(event)); - } - - NPersQueue::TReadResponse resp; - auto& releaseResp = *resp.MutableRelease(); - releaseResp.SetTopic(MakeLegacyTopicName(partitionStream)); - releaseResp.SetPartition(partitionStream->GetPartitionId()); - releaseResp.SetCanCommit(true); - releaseResp.SetGeneration(partitionStream->GetPartitionStreamId()); - AddResponse(std::move(resp)); - } else { - event.Confirm(); - } -} - -void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&& event) { - NPersQueue::TReadResponse resp; - auto& statusResp = *resp.MutablePartitionStatus(); - const auto& partitionStream = event.GetPartitionStream(); - statusResp.SetGeneration(partitionStream->GetPartitionStreamId()); - statusResp.SetTopic(MakeLegacyTopicName(partitionStream)); - statusResp.SetPartition(partitionStream->GetPartitionId()); - statusResp.SetCommittedOffset(event.GetCommittedOffset()); - statusResp.SetEndOffset(event.GetEndOffset()); - statusResp.SetWriteWatermarkMs(event.GetWriteWatermark().MilliSeconds()); - AddResponse(std::move(resp)); -} - -void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&& event) { - const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); - CurrentPartitionStreams.erase(partitionStreamId); - - { - NPersQueue::TReadResponse resp; - auto& respCommit = *resp.MutableCommit(); - for (ui64 cookie : CookiesRequestedToCommit[partitionStreamId]) { - respCommit.AddCookie(cookie); - } - if (resp.GetCommit().CookieSize()) { - AddResponse(std::move(resp)); - } - CookiesRequestedToCommit.erase(partitionStreamId); - } - - { - for (ui64 cookie : PartitionStreamToCookies[partitionStreamId]) { - CookieToOffsets.erase(cookie); - } - PartitionStreamToCookies.erase(partitionStreamId); - } - - { - auto begin = MaxOffsetToCookie.lower_bound(std::make_pair(partitionStreamId, 0)); // The first available offset. - if (begin != MaxOffsetToCookie.end() && begin->first.first == partitionStreamId) { - auto end = begin; - while (end != MaxOffsetToCookie.end() && end->first.first == partitionStreamId) { - ++end; - } - MaxOffsetToCookie.erase(begin, end); - } - } - - if (Settings.UseLockSession) { - PartitionStreamToUncommittedOffsets.erase(partitionStreamId); - UnconfirmedDestroys.erase(partitionStreamId); - - if (event.GetReason() != NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent::EReason::DestroyConfirmedByUser) { - NPersQueue::TReadResponse resp; - auto& releaseResp = *resp.MutableRelease(); - const auto& partitionStream = event.GetPartitionStream(); - releaseResp.SetTopic(MakeLegacyTopicName(partitionStream)); - releaseResp.SetPartition(partitionStream->GetPartitionId()); - releaseResp.SetCanCommit(false); - releaseResp.SetGeneration(partitionStream->GetPartitionStreamId()); - AddResponse(std::move(resp)); - } - } -} - -void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TSessionClosedEvent&& event) { - Destroy(event.GetIssues().ToString(), ToErrorCode(event.GetStatus())); -} - -void TYdbSdkCompatibilityConsumer::AddResponse(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead) { - Responses.emplace(std::move(response), std::move(readyToRead)); -} - -void TYdbSdkCompatibilityConsumer::AddResponse(TReadResponse&& response) { - Responses.emplace(std::move(response)); -} - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h deleted file mode 100644 index 0ff67566866..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h +++ /dev/null @@ -1,75 +0,0 @@ -#pragma once -#include "iconsumer_p.h" - -#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h> - -#include <library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h> - -#include <queue> - -namespace NPersQueue { - -class TYdbSdkCompatibilityConsumer : public IConsumerImpl, - public std::enable_shared_from_this<TYdbSdkCompatibilityConsumer> -{ -public: - TYdbSdkCompatibilityConsumer(const TConsumerSettings& settings, - std::shared_ptr<void> destroyEventRef, - TIntrusivePtr<TPQLibPrivate> pqLib, - TIntrusivePtr<ILogger> logger, - NYdb::NPersQueue::TPersQueueClient& client); - ~TYdbSdkCompatibilityConsumer(); - - void Init() override; - - NThreading::TFuture<TConsumerCreateResponse> Start(TInstant) noexcept override; - NThreading::TFuture<TError> IsDead() noexcept override; - void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; - void Commit(const TVector<ui64>& cookies) noexcept override; - void Cancel() override; - void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; - -private: - class TReadSessionEventVisitor; - - NYdb::NPersQueue::TReadSessionSettings MakeReadSessionSettings(); - void SubscribeToNextEvent(); - void OnReadSessionEvent(); - void AnswerToRequests(); - void Destroy(const TError& description); - void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); - - void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent&& event); - void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&& event); - void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent&& event); - void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent&& event); - void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&& event); - void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&& event); - void HandleEvent(NYdb::NPersQueue::TSessionClosedEvent&& event); - - void AddResponse(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead); - void AddResponse(TReadResponse&& response); - -private: - const TConsumerSettings Settings; - NYdb::NPersQueue::TReadSessionSettings YdbSdkSettings; - TIntrusivePtr<ILogger> Logger; - TString SessionId; - NYdb::NPersQueue::TPersQueueClient& Client; - std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession; - NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); - std::queue<NThreading::TPromise<TConsumerMessage>> Requests; - std::queue<TConsumerMessage> Responses; - bool SubscribedToNextEvent = false; - THashMap<ui64, std::pair<NYdb::NPersQueue::TDeferredCommit, ui64>> CookieToOffsets; // Cookie -> { commit, partition stream id }. - TMap<std::pair<ui64, ui64>, ui64> MaxOffsetToCookie; // { partition stream id, max offset of cookie } -> cookie. - THashMap<ui64, THashSet<ui64>> CookiesRequestedToCommit; // Partition stream id -> cookies that user requested to commit. - THashMap<ui64, THashSet<ui64>> PartitionStreamToCookies; // Partition stream id -> cookies. - ui64 NextCookie = 1; - // Commits for graceful release partition after commit. - THashMap<ui64, TDisjointIntervalTree<ui64>> PartitionStreamToUncommittedOffsets; // Partition stream id -> set of offsets. - THashMap<ui64, NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent> UnconfirmedDestroys; // Partition stream id -> destroy events. - THashMap<ui64, NYdb::NPersQueue::TPartitionStream::TPtr> CurrentPartitionStreams; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h deleted file mode 100644 index 8b429053fdd..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include "types.h" -#include "responses.h" - -#include <library/cpp/threading/future/future.h> - -#include <util/generic/map.h> -#include <util/generic/vector.h> - -namespace NPersQueue { - -struct TPartition { - TString Topic; - ui32 PartitionId = 0; - - bool operator <(const TPartition& other) const { - return std::tie(Topic, PartitionId) < std::tie(other.Topic, other.PartitionId); - } - - explicit TPartition(const TString& topic, const ui32 partitionId) - : Topic(topic) - , PartitionId(partitionId) - {} -}; - -struct TProcessedMessage { - TString Topic; - TString Data; - TString SourceIdPrefix; - ui32 Group = 0; -}; - -struct TProcessedData { - TVector<TProcessedMessage> Messages; // maybe grouped by topic? -}; - -struct TOriginMessage { - TReadResponse::TData::TMessage Message; - NThreading::TPromise<TProcessedData> Processed; -}; - -struct TOriginData { - TMap<TPartition, TVector<TOriginMessage>> Messages; -}; - - -class IProcessor { -public: - virtual ~IProcessor() = default; - - // Returns data and promise for each message. - // Client MUST process each message individually in a deterministic way. - // Each original message CAN be transformed into several messages and written to several topics/sourceids. - // But for any given original message all pairs (topic, sourceIdPrefix) of resulting messages MUST be unique. - // Client MUST finish processing by signaling corresponding promise. - // Otherwise, exactly-once guarantees cannot be hold. - virtual NThreading::TFuture<TOriginData> GetNextData() noexcept = 0; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h deleted file mode 100644 index cbb221e9ee8..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "types.h" -#include "responses.h" -#include <library/cpp/threading/future/future.h> - -namespace NPersQueue { - -// IProducer is threadsafe. -// If one creates several producers with same SourceId in same topic - first Producer (from point of server view) will die. -// There could be only one producer with concrete SourceId in concrete topic at once. - -class IProducer { -public: - // Start producer. - // Producer can be used after its start will be finished. - virtual NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept = 0; - - NThreading::TFuture<TProducerCreateResponse> Start(TDuration timeout) noexcept { - return Start(TInstant::Now() + timeout); - } - - // Add write request to queue. - virtual NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept = 0; - - // Add write request to queue without specifying seqNo. So, without deduplication (at least once guarantee). - virtual NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept = 0; - - // Get future that is signalled when producer is dead. - virtual NThreading::TFuture<TError> IsDead() noexcept = 0; - - virtual ~IProducer() = default; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h deleted file mode 100644 index bb3320d7950..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once -#include <util/generic/ptr.h> -#include <util/generic/string.h> - -namespace NPersQueue { - -class ILogger : public TAtomicRefCount<ILogger> { -public: - virtual ~ILogger() = default; - //level = syslog level - virtual void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) = 0; - virtual bool IsEnabled(int level) const = 0; -}; - -class TCerrLogger : public ILogger { -public: - explicit TCerrLogger(int level) - : Level(level) - { - } - - ~TCerrLogger() override = default; - - void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) override; - - bool IsEnabled(int level) const override - { - return level <= Level; - } - - int GetLevel() const { - return Level; - } - -private: - static TStringBuf LevelToString(int level); - -private: - int Level; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h deleted file mode 100644 index 1b1f7646c8f..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include "types.h" -#include "iproducer.h" -#include "iconsumer.h" -#include "iprocessor.h" -#include "responses.h" - -#include <library/cpp/threading/future/future.h> - -#include <util/generic/noncopyable.h> -#include <util/generic/ptr.h> - -namespace NPersQueue { - -class TPQLibPrivate; - -class TPQLib : public TNonCopyable { -public: - explicit TPQLib(const TPQLibSettings& settings = TPQLibSettings()); - ~TPQLib(); - - // Deprecated flag means that PQLib doen't wait its objects during destruction. - // This behaviour leads to potential thread leakage and thread sanitizer errors. - // It is recommended to specify deprecated = false. In that case PQLib will cancel - // all objects and correctly wait for its threads in destructor. But, unfortunately, - // we can't migrate all current clients to this default behaviour automatically, - // because this can break existing programs unpredictably. - - // Producers creation - THolder<IProducer> CreateProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); - THolder<IProducer> CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); - - // Consumers creation - THolder<IConsumer> CreateConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); - - // Processors creation - THolder<IProcessor> CreateProcessor(const TProcessorSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); - - void SetLogger(TIntrusivePtr<ILogger> logger); - - TString GetUserAgent() const; - void SetUserAgent(const TString& userAgent); - -private: - TIntrusivePtr<TPQLibPrivate> Impl; - TAtomic Alive = 1; // Debug check for valid PQLib usage. -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h deleted file mode 100644 index de5a801172b..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h +++ /dev/null @@ -1,106 +0,0 @@ -#pragma once - -#include "types.h" - -#include <library/cpp/threading/future/future.h> -#include <util/generic/string.h> - -namespace NPersQueue { - -struct TProducerCreateResponse { - TProducerCreateResponse(TWriteResponse&& response) - : Response(std::move(response)) - { - } - - TWriteResponse Response; -}; - -struct TProducerCommitResponse { - TProducerCommitResponse(TProducerSeqNo seqNo, TData data, TWriteResponse&& response) - : Response(std::move(response)) - , Data(std::move(data)) - , SeqNo(seqNo) - { - } - - TWriteResponse Response; - TData Data; - TProducerSeqNo SeqNo; -}; - - -struct TConsumerCreateResponse { - TConsumerCreateResponse(TReadResponse&& response) - : Response(std::move(response)) - { - } - - //will contain Error or Init - TReadResponse Response; -}; - -enum EMessageType { - EMT_LOCK, - EMT_RELEASE, - EMT_DATA, - EMT_ERROR, - EMT_STATUS, - EMT_COMMIT -}; - -struct TLockInfo { - ui64 ReadOffset = 0; - ui64 CommitOffset = 0; - bool VerifyReadOffset = false; - - TLockInfo() = default; - - // compatibility with msvc2015 - TLockInfo(ui64 readOffset, ui64 commitOffset, bool verifyReadOffset) - : ReadOffset(readOffset) - , CommitOffset(commitOffset) - , VerifyReadOffset(verifyReadOffset) - {} -}; - -static EMessageType GetType(const TReadResponse& response) { - if (response.HasData()) return EMT_DATA; - if (response.HasError()) return EMT_ERROR; - if (response.HasCommit()) return EMT_COMMIT; - if (response.HasPartitionStatus()) return EMT_STATUS; - if (response.HasRelease()) return EMT_RELEASE; - if (response.HasLock()) return EMT_LOCK; - - // for no warn return anything. - return EMT_LOCK; -} - - - -struct TConsumerMessage { - EMessageType Type; - //for DATA/ERROR/LOCK/RELEASE/COMMIT/STATUS: - //will contain Error, Data, Lock or Release and be consistent with Type - TReadResponse Response; - - //for LOCK only: - mutable NThreading::TPromise<TLockInfo> ReadyToRead; - - TConsumerMessage(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead) - : Type(EMT_LOCK) - , Response(std::move(response)) - , ReadyToRead(std::move(readyToRead)) - { - Y_VERIFY(GetType(Response) == EMT_LOCK); - } - - - explicit TConsumerMessage(TReadResponse&& response) - : Type(GetType(response)) - , Response(std::move(response)) - { - } -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp deleted file mode 100644 index b30115e2943..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp +++ /dev/null @@ -1,236 +0,0 @@ -#include <util/datetime/base.h> -#include <util/stream/file.h> -#include <util/string/join.h> -#include <util/string/vector.h> - -#include <google/protobuf/message.h> - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -using namespace NPersQueue; - - -int main() { - - // [BEGIN create pqlib] - // Step 1. First create logger. It could be your logger inherited from ILogger interface or one from lib. - TIntrusivePtr<ILogger> logger = new TCerrLogger(7); - - // Step 2. Create settings of main TPQLib object. - TPQLibSettings pqLibSettings; - - pqLibSettings.DefaultLogger = logger; - - // Step 3. Create TPQLib object. - TPQLib pq(pqLibSettings); - // [END create pqlib] - - // [BEGIN create consumer] - // Step 4. Parameters used in sample. - - // List of topics to read. - TVector<TString> topics = {"topic_path"}; - - // Path of consumer that will read topics. - TString consumerPath = "consumer_path"; - - // OAuth token. - TString oauthToken = "oauth_token"; - - // IAM JWT key - TString jwtKey = "<jwt json key>"; - - // TVM parameters. - TString tvmSecret = "tvm_secret"; - ui32 tvmClientId = 200000; - ui32 tvmDstId = 2001147; // logbroker main prestable - tvmDstId = 2001059; // logbroker main production - - // Logbroker endpoint. - TString endpoint = "man.logbroker.yandex.net"; - - // Simple single read. - - // Step 5. Create settings of consumer object. - TConsumerSettings settings; - // Fill consumer settings. See types.h for more information. - // Settings without defaults, must be set always. - settings.Server.Address = endpoint; - { - // Working with logbroker in Yandex.Cloud: - // Fill database for Yandex.Cloud Logbroker. - settings.Server.Database = "/Root"; - // Enable TLS. - settings.Server.EnableSecureConnection(""); - } - settings.Topics = topics; - settings.ClientId = consumerPath; - - // One must set CredentialsProvider for authentification. There could be OAuth and TVM providers. - // Create OAuth provider from OAuth ticket. - settings.CredentialsProvider = CreateOAuthCredentialsProvider(oauthToken); - // Or create TVM provider from TVM settings. - settings.CredentialsProvider = CreateTVMCredentialsProvider(tvmSecret, tvmClientId, tvmDstId, logger); - // Or create IAM provider from jwt-key - settings.CredentialsProvider = CreateIAMJwtParamsCredentialsForwarder(jwtKey, logger); - - // Step 6. Create consumer object. PQLib object must be valid during all lifetime of consumer, otherwise reading from consumer will fail but this is not UB. - auto consumer = pq.CreateConsumer(settings, logger); - - // Step 7. Start consuming data. - auto future = consumer->Start(); - - // Step 9. Wait initialization and check it result. - future.Wait(); - - if (future.GetValue().Response.HasError()) { - return 1; - } - // [END create consumer] - - // [BEGIN read messages] - // Step 10. Simple single read. - // Step 10.1. Request message from consumer. - auto msg = consumer->GetNextMessage(); - - // Step 10.2. Wait for message. - msg.Wait(); - - auto value = msg.GetValue(); - - // Step 10.3. Check result. - switch (value.Type) { - // Step 10.3.1. Result is error - consumer is in incorrect state now. You can log error and recreate everything starting from step 5. - case EMT_ERROR: - return 1; - // Step 10.3.2. Read data. - case EMT_DATA: { - // Process received messages. Inside will be several batches. - for (const auto& t : value.Response.data().message_batch()) { - const TString topic = t.topic(); - const ui32 partition = t.partition(); - Y_UNUSED(topic); - Y_UNUSED(partition); - // There are several messages from partition of topic inside one batch. - for (const auto& m : t.message()) { - const ui64 offset = m.offset(); - const TString& data = m.data(); - // Each message has offset in partition, payload data (decompressed or as is) and metadata. - Y_UNUSED(offset); - Y_UNUSED(data); - } - } - - // Step 10.3.3. Each processed data batch from step 10.3.2 must be committed once all data inside is processed. - - // This must be done using cookie from data batch. - auto cookie = msg.GetValue().Response.GetData().GetCookie(); - // Commit cookie['s]. Acknowledge from server will be received inside one of next messages from consumer later. - consumer->Commit({cookie}); - break; - } - // Step 10.3.4. Commit acknowledge. Can be used for logging and debugging purposes. (Can't be first message, only possible if one makes several - // consecutive reads. - case EMT_COMMIT: - break; - // Handle other cases - only for compiling, there could not be any. - default: - break; - - } - // [END read messages] - - // [BEGIN advanced read messages] - - // Point that consumer will perform advanced reads. - settings.UseLockSession = true; - - { - // Step 11. Create consumer object. PQLib object must be valid during all lifetime of consumer, otherwise reading from consumer will fail but this is not UB. - auto consumer = pq.CreateConsumer(settings, logger); - - // Step 12. Start consuming data. - auto future = consumer->Start(); - - // Step 14. Wait initialization and check it result. - future.Wait(); - - if (future.GetValue().Response.HasError()) { - return 1; - } - - // Step 14. Event loop. - while (true) { - // Step 14.1. Request message from consumer. - auto msg = consumer->GetNextMessage(); - - // Step 14.2. Wait for message. - msg.Wait(); - - const auto& value = msg.GetValue(); - - // Step 14.3. Check result. - switch (value.Type) { - // Step 14.3.1. Result is error: consumer is in incorrect state now. You can log error and recreate everything starting from step 11. - case EMT_ERROR: - return 1; - // Step 14.3.2. Lock request from server. - case EMT_LOCK: { - const TString& topic = value.Response.lock().topic(); - const ui32 partition = value.Response.lock().partition(); - // Server is ready to send data from corresponding partition of topic. Topic is legacy topic name = rt3.<cluster>--<converted topic path> - Y_UNUSED(topic); - Y_UNUSED(partition); - // One must react on this event with setting promise ReadyToRead right now or later. Only after this client will start receiving data from this partition. - msg.GetValue().ReadyToRead.SetValue(TLockInfo{0 /*readOffset*/, 0 /*commitOffset*/, false /*verifyReadOffset*/}); - break; - } - // Step 14.3.3. Release request from server. - case EMT_RELEASE: { - const TString& topic = value.Response.release().topic(); - const ui32 partition = value.Response.release().partition(); - Y_UNUSED(topic); - Y_UNUSED(partition); - // // Server will not send any data from partition of topic until next Lock request is confirmed by client. - break; - } - // Step 14.3.4. Read data. - case EMT_DATA: { - // Process received messages. Inside will be several batches. The same as in the simple read. - for (const auto& t : value.Response.data().message_batch()) { - const TString topic = t.topic(); - const ui32 partition = t.partition(); - Y_UNUSED(topic); - Y_UNUSED(partition); - // There are several messages from partition of topic inside one batch. - for (const auto& m : t.message()) { - const ui64 offset = m.offset(); - const TString& data = m.data(); - // Each message has offset in partition, payload data (decompressed or as is) and metadata. - Y_UNUSED(offset); - Y_UNUSED(data); - } - } - - // Step 14.3.5. Each processed data batch from step 14.3.4 should be committed after all data inside is processed. - - // This must be done using cookie from data batch. - auto cookie = msg.GetValue().Response.GetData().GetCookie(); - // Commit cookie['s]. Acknowledge from server will be received inside message from consumer later. - consumer->Commit({cookie}); - break; - } - // Step 14.3.6. Commit acknowledge. Can be used for logging and debugging purposes. - case EMT_STATUS: - case EMT_COMMIT: - break; - } // switch() - } // while() - } - - // [END advanced read messages] - - // Step 15. End of working. Nothing special needs to be done - desctuction of objects only. - - return 0; -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp deleted file mode 100644 index c91838892b1..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include <queue> - -#include <util/datetime/base.h> -#include <util/stream/file.h> -#include <util/string/join.h> -#include <util/string/vector.h> - -#include <google/protobuf/message.h> - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -using namespace NPersQueue; - - -int main() { - - - // [BEGIN create pqlib] - // Step 1. First create logger. It could be your logger inherited from ILogger interface or one from lib. - TIntrusivePtr<ILogger> logger = new TCerrLogger(7); - - // Step 2. Create settings of main TPQLib object. - TPQLibSettings pqLibSettings; - - // Number of threads for processing gRPC events. - pqLibSettings.ThreadsCount = 1; - // Number of threads for compression/decomression actions. - pqLibSettings.CompressionPoolThreads = 3; - // Default logger. Used inside consumers and producers if no logger provided. - pqLibSettings.DefaultLogger = logger; - - // Step 3. Create TPQLib object. - TPQLib pq(pqLibSettings); - // [END create pqlib] - - // [BEGIN create producer] - // Step 4. Initialize producer parameters - - // Topics to read. - TString topic = "topic_path"; - // Partitions group. 0 means any. - ui32 group = 0; - - // Source identifier of messages group. - TString sourceId = "source_id"; - - // OAuth token. - TString oauthToken = "oauth_token"; - - // IAM JWT key - TString jwtKey = "<jwt json key>"; - - // TVM parameters. - TString tvmSecret = "tvm_secret"; - ui32 tvmClientId = 200000; - ui32 tvmDstId = 2001147; // logbroker main prestable - tvmDstId = 2001059; // logbroker main production - - // Logbroker endpoint. - TString endpoint = "logbroker.yandex.net"; - - // Step 5. Put producer parameters into settings. - TProducerSettings settings; - // Logbroker endpoint. - settings.Server.Address = endpoint; - { - // Working with logbroker in Yandex.Cloud: - // Fill database for Yandex.Cloud Logbroker. - settings.Server.Database = "/Root"; - // Enable TLS. - settings.Server.EnableSecureConnection(""); - } - // Topic path. - settings.Topic = topic; - // SourceId of messages group. - settings.SourceId = sourceId; - // Partition group to write to. - settings.PartitionGroup = group; - // ReconnectOnFailure will gurantee that producer will retry writes in case of failre. - settings.ReconnectOnFailure = true; //retries on server errors - // Codec describes how to compress data. - settings.Codec = NPersQueueCommon::ECodec::GZIP; - - // One must set CredentialsProvider for authentification. There could be OAuth and TVM providers. - // Create OAuth provider from OAuth ticket. - settings.CredentialsProvider = CreateOAuthCredentialsProvider(oauthToken); - // Or create TVM provider from TVM settings. - settings.CredentialsProvider = CreateTVMCredentialsProvider(tvmSecret, tvmClientId, tvmDstId, logger); - // Or create IAM provider from jwt-key - settings.CredentialsProvider = CreateIAMJwtParamsCredentialsForwarder(jwtKey, logger); - - // Step 6. Create producer object. PQLib object must be valid during all lifetime of producer, otherwise writing to producer will fail but this is not UB. - auto producer = pq.CreateProducer(settings, logger); - - // Step 7. Start producer. - auto future = producer->Start(); - - // Step 8. Wait initialization and check it result. - future.Wait(); - - if (future.GetValue().Response.HasError()) { - return 1; - } - // [END create producer] - - // [BEGIN write message] - - // Step 9. Single write sample. - // Step 9.1. Form message. - // Creation timestamp. - TInstant timestamp = TInstant::Now(); - // Message payload. - TString payload = "abacaba"; - - TData data{payload, timestamp}; - // Corresponding messages's sequence number. - ui64 seqNo = 1; - - // Step 9.2. Write message. Until result future is signalled - message is stored inside PQLib memory. Future will be signalled with message payload and result of writing. - auto result = producer->Write(seqNo, data); - - // Step 9.3 Check writing result. - // Check all results. - result.Wait(); - Y_VERIFY(!result.GetValue().Response.HasError()); - - // Only when future is destroyed no memory is used by message. - - // [END write message] - - // [BEGIN write messages with inflight] - - // Step 10. Writing loop with in-flight. - // Max in-flight border, parameter. - ui32 maxInflyCount = 10; - // Queue for messages in-flight. - std::queue<NThreading::TFuture<TProducerCommitResponse>> v; - - while(true) { - // Creation timestamp. - TInstant timestamp = TInstant::Now(); - // Message payload. - TString payload = "abacaba"; - TData data{payload, timestamp}; - - // SeqNo must increase for consecutive messages. - ++seqNo; - - // Step 10.1. Write message. - auto result = producer->Write(seqNo, data); - // Step 10.2. Put future inside queue. - v.push(result); - // Step 10.3. If in-flight size is too big. - if (v.size() > maxInflyCount) { - // Step 10.4. Wait for first message in queue processing result, check it. - v.front().Wait(); - Y_VERIFY(!v.front().GetValue().Response.HasError()); - // Step 10.5. Drop first processed message. Only after memory of this message will be freed. - v.pop(); - } - } - - // Step 10.6. After processing loop one must wait for all not yet processed messages. - - while (!v.empty()) { - v.front().Wait(); - Y_VERIFY(!v.front().GetValue().Response.HasError()); - v.pop(); - } - - // [END write messages with inflight] - - return 0; -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h deleted file mode 100644 index 37c4423d6ff..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h +++ /dev/null @@ -1,402 +0,0 @@ -#pragma once -#include "credentials_provider.h" -#include "logger.h" -#include <kikimr/yndx/api/protos/persqueue.pb.h> - - -#include <library/cpp/logger/priority.h> - -#include <util/generic/string.h> -#include <util/generic/buffer.h> -#include <util/generic/maybe.h> -#include <util/generic/vector.h> -#include <util/generic/hash.h> -#include <util/datetime/base.h> -#include <util/stream/output.h> -#include <util/string/builder.h> - -#include <grpc++/channel.h> - -#include <limits> - -namespace NPersQueue { -using ECodec = NPersQueueCommon::ECodec; -using TCredentials = NPersQueueCommon::TCredentials; -using TError = NPersQueueCommon::TError; - -using TProducerSeqNo = ui64; // file offset or something with equivalent meaning. -using TConsumerOffset = i64; // partition offset. Have special value -1. All other values are >= 0. -using TCredProviderPtr = std::shared_ptr<ICredentialsProvider>; - -constexpr ui32 DEFAULT_MEMORY_USAGE = 100 << 20; //100mb -constexpr ui32 DEFAULT_READS_INFLY = 10; -constexpr ui32 DEFAULT_BATCH_SIZE = 1 << 20; //1mb -constexpr TDuration DEFAULT_CHANNEL_CREATION_TIMEOUT = TDuration::Seconds(10); -constexpr TDuration DEFAULT_RECONNECTION_DELAY = TDuration::MilliSeconds(10); -constexpr TDuration DEFAULT_MAX_RECONNECTION_DELAY = TDuration::Seconds(1); -constexpr TDuration DEFAULT_START_SESSION_TIMEOUT = TDuration::Seconds(30); -constexpr TDuration DEFAULT_CHECK_TOKEN_PERIOD = TDuration::Hours(1); -inline const THashMap<ECodec, TString> CodecIdByType{{ECodec::DEFAULT, TString(1, '\0')}, {ECodec::RAW, TString(1, '\0')}, {ECodec::GZIP, TString(1, '\1')}, {ECodec::LZOP, TString(1, '\2')}, {ECodec::ZSTD, TString(1, '\3')}}; - -struct TPQLibSettings { - size_t ThreadsCount = 1; - size_t CompressionPoolThreads = 2; - size_t GRpcThreads = 1; - TDuration ChannelCreationTimeout = DEFAULT_CHANNEL_CREATION_TIMEOUT; - TIntrusivePtr<ILogger> DefaultLogger = {}; - - bool EnableGRpcV1 = std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED"); -}; - -enum class EClusterDiscoveryUsageMode { - Auto, - Use, - DontUse, -}; - -// destination server -struct TServerSetting { - // Endpoint of logbroker cluster to connect. - // Possible variants: - // - // 1. lbkx cluster. - // Set address to lbkx.logbroker.yandex.net - // - // 2. logbroker federation. - // If you want to write/read to/from concrete cluster - // (with ReadMirroredPartitions or not, but without ReadFromAllClusterSources option), - // use the following addresses: - // iva.logbroker.yandex.net - // man.logbroker.yandex.net - // myt.logbroker.yandex.net - // sas.logbroker.yandex.net - // vla.logbroker.yandex.net - // - // If you create consumer with ReadFromAllClusterSources option, - // use logbroker.yandex.net - TString Address; - - ui16 Port = 2135; - TString Database = GetRootDatabase(); - EClusterDiscoveryUsageMode UseLogbrokerCDS = EClusterDiscoveryUsageMode::Auto; - bool UseSecureConnection = false; - TString CaCert = TString(); - - TServerSetting(TString address, ui16 port = 2135) - : Address(std::move(address)) - , Port(port) - { - Y_VERIFY(!Address.empty()); - Y_VERIFY(Port > 0); - } - - TServerSetting() = default; - TServerSetting(const TServerSetting&) = default; - TServerSetting(TServerSetting&&) = default; - - TServerSetting& operator=(const TServerSetting&) = default; - TServerSetting& operator=(TServerSetting&&) = default; - - TString GetFullAddressString() const { - return TStringBuilder() << Address << ':' << Port; - } - void EnableSecureConnection(const TString& caCert) { - UseSecureConnection = true; - CaCert = caCert; - } - void DisableSecureConnection() { - UseSecureConnection = false; - CaCert.clear(); - } - - const TString& GetRootDatabase(); - - bool operator < (const TServerSetting& rhs) const { - return std::tie(Address, Port, Database, UseSecureConnection) < std::tie(rhs.Address, rhs.Port, rhs.Database, rhs.UseSecureConnection); - } -}; - -// -// consumer/producer settings -// - -struct TProducerSettings { - TServerSetting Server; - std::shared_ptr<ICredentialsProvider> CredentialsProvider = CreateInsecureCredentialsProvider(); - - TString Topic; - TString SourceId; - - ui32 PartitionGroup = 0; - ECodec Codec = ECodec::GZIP; - int Quality = -1; - THashMap<TString, TString> ExtraAttrs; - - // Should we reconnect when session breaks? - bool ReconnectOnFailure = false; - - // - // Setting for reconnection - // - - // In case of lost connection producer will try - // to reconnect and resend data <= MaxAttempts times. - // Then producer will be dead. - unsigned MaxAttempts = std::numeric_limits<unsigned>::max(); - - // Time delay before next producer reconnection attempt. - // The second attempt will be in 2 * ReconnectionDelay, - // the third - in 4 * ReconnectionDelay, etc. Maximum value - // of delay is MaxReconnectionDelay. - TDuration ReconnectionDelay = DEFAULT_RECONNECTION_DELAY; - TDuration MaxReconnectionDelay = DEFAULT_MAX_RECONNECTION_DELAY; - - // Timeout for session initialization operation. - TDuration StartSessionTimeout = DEFAULT_START_SESSION_TIMEOUT; - // Producer will check token from 'ICredentialsProvider' with this period and if changed - send to server as soon as possible. - // Available only with gRPC data plane API v1. - TDuration CheckTokenPeriod = DEFAULT_CHECK_TOKEN_PERIOD; - - // Preferred cluster to write. - // This setting is used only in case of cluster discovery is used. - // Case insensitive. - // If PreferredCluster is not available, - // producer will connect to other cluster. - TString PreferredCluster; - - bool DisableCDS = false; // only for grpcv1 mode -}; - -struct TMultiClusterProducerSettings { - struct TServerWeight { - // Setting of producer connected to concrete DC. - // ReconnectOnFailure must be true. - // MaxAttempts must be a finite number. - // SourceId will be complemented with source id prefix and producer type. Can be empty. - // There is no guarantee that messages will arrive in the order of writes in case of one DC is absent. - TProducerSettings ProducerSettings; - unsigned Weight; - - explicit TServerWeight(TProducerSettings producerSettings, unsigned weight = 1) - : ProducerSettings(std::move(producerSettings)) - , Weight(weight) - { - Y_VERIFY(Weight > 0); - } - - TServerWeight() = default; - TServerWeight(const TServerWeight&) = default; - TServerWeight(TServerWeight&&) = default; - - TServerWeight& operator=(const TServerWeight&) = default; - TServerWeight& operator=(TServerWeight&&) = default; - }; - - std::vector<TServerWeight> ServerWeights; - size_t MinimumWorkingDcsCount = 1; // Minimum Dcs to start and not to die. - TString SourceIdPrefix; // There is no guarantee that messages will arrive in the order of writes in case of one DC is absent. -}; - -struct TConsumerSettings { - TVector<TString> Topics; - bool ReadMirroredPartitions = true; - - // Discovers all clusters where topic is presented and create consumer that reads from all of them. - // Conflicts with ReadMirroredPartitions option (should be set to "no"). - // Will ignore the following options: - // ReconnectOnFailure (forced to "yes") - // MaxAttempts (forced to infinity == std::numeric_limits<unsigned>::max()) - // Server.UseLogbrokerCDS (forced to "Use") - // - // Also see description of option ReconnectOnFailure. - bool ReadFromAllClusterSources = false; - - TServerSetting Server; - std::shared_ptr<ICredentialsProvider> CredentialsProvider = CreateInsecureCredentialsProvider(); - TString ClientId; - - bool UseLockSession = false; - ui32 PartsAtOnce = 0; - - bool Unpack = true; - bool SkipBrokenChunks = false; - - ui64 MaxMemoryUsage = DEFAULT_MEMORY_USAGE; - ui32 MaxInflyRequests = DEFAULT_READS_INFLY; - - // read settings - ui32 MaxCount = 0; // zero means unlimited - ui32 MaxSize = DEFAULT_BATCH_SIZE; // zero means unlimited - ui32 MaxTimeLagMs = 0; // zero means unlimited - ui64 ReadTimestampMs = 0; // read data from this timestamp - - ui32 MaxUncommittedCount = 0; // zero means unlimited - ui64 MaxUncommittedSize = 0; // zero means unlimited - - TVector<ui32> PartitionGroups; // groups to read - - bool PreferLocalProxy = false; - bool BalanceRightNow = false; - bool CommitsDisabled = false; - - // Should we reconnect when session breaks? - // - // Keep in mind that it is possible to receive - // message duplicates when reconnection occures. - bool ReconnectOnFailure = false; - - // - // Settings for reconnection - // - - // In case of lost connection consumer will try - // to reconnect and resend requests <= MaxAttempts times. - // Then consumer will be dead. - unsigned MaxAttempts = std::numeric_limits<unsigned>::max(); - - // Time delay before next consumer reconnection attempt. - // The second attempt will be in 2 * ReconnectionDelay, - // the third - in 3 * ReconnectionDelay, etc. Maximum value - // of delay is MaxReconnectionDelay. - TDuration ReconnectionDelay = DEFAULT_RECONNECTION_DELAY; - TDuration MaxReconnectionDelay = DEFAULT_MAX_RECONNECTION_DELAY; - TDuration StartSessionTimeout = DEFAULT_START_SESSION_TIMEOUT; - - bool DisableCDS = false; // only for grpcv1 mode - - bool UseV2RetryPolicyInCompatMode = false; -}; - -struct TProcessorSettings { - ui64 MaxOriginDataMemoryUsage; - ui64 MaxProcessedDataMemoryUsage; - TDuration SourceIdIdleTimeout; - - std::shared_ptr<ICredentialsProvider> CredentialsProvider; - - //UseLockSession parameter will be ignored - TConsumerSettings ConsumerSettings; - - //Topic and SourceId pararameters will be ignored - TProducerSettings ProducerSettings; - - TProcessorSettings() - : MaxOriginDataMemoryUsage(DEFAULT_MEMORY_USAGE) - , MaxProcessedDataMemoryUsage(DEFAULT_MEMORY_USAGE) - , SourceIdIdleTimeout(TDuration::Hours(1)) - , CredentialsProvider(CreateInsecureCredentialsProvider()) - { - ConsumerSettings.CredentialsProvider = ProducerSettings.CredentialsProvider = CredentialsProvider; - } -}; - -class TData { -public: - // empty: not for passing to producer - TData() = default; - - // implicit constructor from source data to be encoded later by producer with default codec from settings - // or explicitly specified one - TData(TString sourceData, ECodec codec = ECodec::DEFAULT, TInstant timestamp = TInstant::Now()) - : TData(timestamp, sourceData, codec == ECodec::RAW ? sourceData : TString(), codec, sourceData.size()) - { - Y_VERIFY(!SourceData.empty()); - Y_VERIFY(timestamp != TInstant::Zero()); - } - - TData(TString sourceData, TInstant timestamp) - : TData(std::move(sourceData), ECodec::DEFAULT, timestamp) - { - } - - // construct already encoded TData - static TData Encoded(TString encodedData, ECodec codec, TInstant timestamp = TInstant::Now(), ui32 originalSize = 0) { - Y_VERIFY(!encodedData.empty()); - Y_VERIFY(codec != ECodec::RAW && codec != ECodec::DEFAULT); - Y_VERIFY(timestamp != TInstant::Zero()); - return TData(timestamp, TString(encodedData), encodedData, codec, originalSize == 0 ? encodedData.size() * 10 : originalSize); - } - - // construct user defined raw data that shouldn't be encoded by producer - static TData Raw(TString rawData, TInstant timestamp = TInstant::Now()) { - Y_VERIFY(!rawData.empty()); - Y_VERIFY(timestamp != TInstant::Zero()); - return TData(timestamp, rawData, rawData, ECodec::RAW, rawData.size()); - } - - TData(const TData&) = default; - TData(TData&&) = default; - - TData& operator=(const TData&) = default; - TData& operator=(TData&&) = default; - - ECodec GetCodecType() const { - return Codec; - } - - bool IsEncoded() const { - return !EncodedData.empty(); - } - - const TString& GetEncodedData() const { - Y_VERIFY(IsEncoded()); - return EncodedData; - } - - // gets some data for error report - const TString& GetSourceData() const { - return SourceData; - } - - TInstant GetTimestamp() const { - return Timestamp; - } - - bool Empty() const { - return GetTimestamp() == TInstant::Zero(); - } - - // encoding of data by default codec or data codec that is not encoded - // if data's codec is default, uses defaultCodec parameter, - // otherwise uses data's codec - static TData Encode(TData data, ECodec defaultCodec, int quality = -1); - - ui32 GetOriginalSize() const { - return OriginalSize; - } - - // special function to make raw enconding that doesn't need cpu time. - static TData MakeRawIfNotEncoded(TData data); - - bool operator==(const TData& data) const { - return Codec == data.Codec - && Timestamp == data.Timestamp - && SourceData == data.SourceData - && EncodedData == data.EncodedData; - } - - bool operator!=(const TData& data) const { - return !operator==(data); - } - -private: - TData(TInstant timestamp, TString sourceData, TString encodedData, ECodec codec, ui32 originalSize) - : Timestamp(timestamp) - , SourceData(std::move(sourceData)) - , EncodedData(std::move(encodedData)) - , Codec(codec) - , OriginalSize(originalSize) - { - } - - static THolder<IOutputStream> CreateCoder(ECodec codec, TData& data, int quality); - -private: - TInstant Timestamp; - TString SourceData; - TString EncodedData; - ECodec Codec = ECodec::DEFAULT; - ui32 OriginalSize = 0; -}; - -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt deleted file mode 100644 index ec1f7fb67f8..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ - -# This file was gererated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_library(cpp-v2-ut_utils) -target_compile_options(cpp-v2-ut_utils PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_link_libraries(cpp-v2-ut_utils PUBLIC - contrib-libs-cxxsupp - yutil - deprecated-cpp-v2 - yndx-grpc_services-persqueue - yndx-persqueue-msgbus_server - cpp-grpc-server - cpp-testing-unittest - core-client-server - ydb-core-testlib - library-persqueue-topic_parser_public -) -target_sources(cpp-v2-ut_utils PRIVATE - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp - ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp -) diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp deleted file mode 100644 index 6f201f39f50..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp +++ /dev/null @@ -1,263 +0,0 @@ -#include "data_writer.h" - -namespace NPersQueue::NTests { -using namespace NKikimr; - -void TPQDataWriter::Read( - const TString& topic, const TString& clientId, const TString& ticket, bool error, bool checkACL, - bool useBatching, bool onlyCreate -) { - - grpc::ClientContext context; - - auto stream = StubP_->ReadSession(&context); - UNIT_ASSERT(stream); - - // Send initial request. - TReadRequest req; - TReadResponse resp; - - req.MutableInit()->AddTopics(topic); - - req.MutableInit()->SetClientId(clientId); - req.MutableInit()->SetProxyCookie(ProxyCookie); - if (!ticket.empty()) { - req.MutableCredentials()->SetTvmServiceTicket(ticket); - } - - if (useBatching) { - req.MutableInit()->SetProtocolVersion(TReadRequest::Batching); - } - - if (!stream->Write(req)) { - ythrow yexception() << "write fail"; - } - //TODO[komels]: why this leads to timeout? - //Server.AnnoyingClient->GetClientInfo({topic}, clientId, true); - UNIT_ASSERT(stream->Read(&resp)); - if (error) { - UNIT_ASSERT(resp.HasError()); - return; - } - UNIT_ASSERT_C(resp.HasInit(), resp); - - if (onlyCreate) - return; - - for (ui32 i = 0; i < 11; ++i) { - TReadRequest req; - - req.MutableRead()->SetMaxCount(1); - - if (!stream->Write(req)) { - ythrow yexception() << "write fail"; - } - Server.AnnoyingClient->AlterTopic(FullTopicName, i < 10 ? 2 : 3); - - } - - if (checkACL) { - NACLib::TDiffACL acl; - acl.RemoveAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, clientId + "@" BUILTIN_ACL_DOMAIN); - Server.AnnoyingClient->ModifyACL("/Root/PQ", FullTopicName, acl.SerializeAsString()); - - TReadRequest req; - req.MutableRead()->SetMaxCount(1); - if (!stream->Write(req)) { - ythrow yexception() << "write fail"; - } - - UNIT_ASSERT(stream->Read(&resp)); - UNIT_ASSERT(resp.HasError() && resp.GetError().GetCode() == NPersQueue::NErrorCode::EErrorCode::ACCESS_DENIED); - return; - } - Server.AnnoyingClient->GetClientInfo({FullTopicName}, clientId, true); - for (ui32 i = 0; i < 11; ++i) { - TReadResponse resp; - - UNIT_ASSERT(stream->Read(&resp)); - - if (useBatching) { - UNIT_ASSERT(resp.HasBatchedData()); - UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().PartitionDataSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).BatchSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).MessageDataSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).GetMessageData(0).GetOffset(), i); - } else { - UNIT_ASSERT(resp.HasData()); - UNIT_ASSERT_VALUES_EQUAL(resp.GetData().MessageBatchSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(resp.GetData().GetMessageBatch(0).MessageSize(), 1); - UNIT_ASSERT_VALUES_EQUAL(resp.GetData().GetMessageBatch(0).GetMessage(0).GetOffset(), i); - } - } - //TODO: check here that read will never done UNIT_ASSERT(!stream->Read(&resp)); - { - for (ui32 i = 1; i < 11; ++i) { - TReadRequest req; - - req.MutableCommit()->AddCookie(i); - - if (!stream->Write(req)) { - ythrow yexception() << "write fail"; - } - } - ui32 i = 1; - while (i <= 10) { - TReadResponse resp; - - UNIT_ASSERT(stream->Read(&resp)); - Cerr << resp << "\n"; - UNIT_ASSERT(resp.HasCommit()); - UNIT_ASSERT(resp.GetCommit().CookieSize() > 0); - for (ui32 j = 0; j < resp.GetCommit().CookieSize(); ++j) { - UNIT_ASSERT( resp.GetCommit().GetCookie(j) == i); - ++i; - UNIT_ASSERT(i <= 11); - } - } - Server.AnnoyingClient->GetClientInfo({FullTopicName}, clientId, true); - } -} - -void TPQDataWriter::WaitWritePQServiceInitialization() { - TWriteRequest req; - TWriteResponse resp; - while (true) { - grpc::ClientContext context; - - auto stream = StubP_->WriteSession(&context); - UNIT_ASSERT(stream); - req.MutableInit()->SetTopic(ShortTopicName); - req.MutableInit()->SetSourceId("12345678"); - req.MutableInit()->SetProxyCookie(ProxyCookie); - - if (!stream->Write(req)) { - continue; - } - UNIT_ASSERT(stream->Read(&resp)); - if (resp.GetError().GetCode() == NPersQueue::NErrorCode::INITIALIZING) { - Sleep(TDuration::MilliSeconds(50)); - } else { - break; - } - } -} - -ui32 TPQDataWriter::InitSession(const TString& sourceId, ui32 pg, bool success, ui32 step) { - TWriteRequest req; - TWriteResponse resp; - - grpc::ClientContext context; - - auto stream = StubP_->WriteSession(&context); - - UNIT_ASSERT(stream); - req.MutableInit()->SetTopic(ShortTopicName); - req.MutableInit()->SetSourceId(sourceId); - req.MutableInit()->SetPartitionGroup(pg); - req.MutableInit()->SetProxyCookie(ProxyCookie); - - UNIT_ASSERT(stream->Write(req)); - UNIT_ASSERT(stream->Read(&resp)); - Cerr << "Init result: " << resp << "\n"; - //TODO: ensure topic creation - proxy already knows about new partitions, but tablet - no! - if (!success) { - UNIT_ASSERT(resp.HasError()); - return 0; - } else { - if (!resp.HasInit() && step < 5) { - Sleep(TDuration::MilliSeconds(100)); - return InitSession(sourceId, pg, success, step + 1); - } - UNIT_ASSERT(resp.HasInit()); - return resp.GetInit().GetPartition(); - } - return 0; -} - -ui32 TPQDataWriter::WriteImpl( - const TString& topic, const TVector<TString>& data, bool error, const TString& ticket, bool batch -) { - grpc::ClientContext context; - - auto stream = StubP_->WriteSession(&context); - UNIT_ASSERT(stream); - - // Send initial request. - TWriteRequest req; - TWriteResponse resp; - - req.MutableInit()->SetTopic(topic); - req.MutableInit()->SetSourceId(SourceId_); - req.MutableInit()->SetProxyCookie(ProxyCookie); - if (!ticket.empty()) - req.MutableCredentials()->SetTvmServiceTicket(ticket); - auto item = req.MutableInit()->MutableExtraFields()->AddItems(); - item->SetKey("key"); - item->SetValue("value"); - - if (!stream->Write(req)) { - ythrow yexception() << "write fail"; - } - - ui32 part = 0; - - UNIT_ASSERT(stream->Read(&resp)); - - if (!error) { - UNIT_ASSERT_C(resp.HasInit(), resp); - UNIT_ASSERT_C(!resp.GetInit().GetSessionId().empty(), resp); - part = resp.GetInit().GetPartition(); - } else { - Cerr << resp << "\n"; - UNIT_ASSERT(resp.HasError()); - return 0; - } - - // Send data requests. - Flush(data, stream, ticket, batch); - - Flush(data, stream, ticket, batch); - - Flush(data, stream, ticket, batch); - - Flush(data, stream, ticket, batch); - - //will cause only 4 answers in stream->Read - third call will fail, not blocks - stream->WritesDone(); - - UNIT_ASSERT(!stream->Read(&resp)); - - auto status = stream->Finish(); - UNIT_ASSERT(status.ok()); - return part; -} - -ui64 TPQDataWriter::ReadCookieFromMetadata(const std::multimap<grpc::string_ref, grpc::string_ref>& meta) const { - auto ci = meta.find("cookie"); - if (ci == meta.end()) { - ythrow yexception() << "server didn't provide the cookie"; - } else { - return FromString<ui64>(TStringBuf(ci->second.data(), ci->second.size())); - } -} - -void TPQDataWriter::InitializeChannel() { - Channel_ = grpc::CreateChannel("[::1]:" + ToString(Server.GrpcPort), grpc::InsecureChannelCredentials()); - Stub_ = NKikimrClient::TGRpcServer::NewStub(Channel_); - - grpc::ClientContext context; - NKikimrClient::TChooseProxyRequest request; - NKikimrClient::TResponse response; - auto status = Stub_->ChooseProxy(&context, request, &response); - UNIT_ASSERT(status.ok()); - Cerr << response << "\n"; - UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK); - ProxyCookie = response.GetProxyCookie(); - Channel_ = grpc::CreateChannel( - "[" + response.GetProxyName() + "]:" + ToString(Server.GrpcPort), - grpc::InsecureChannelCredentials() - ); - StubP_ = NPersQueue::PersQueueService::NewStub(Channel_); -} -} // namespace NKikimr::NPersQueueTests diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h deleted file mode 100644 index c4e46ab08eb..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h +++ /dev/null @@ -1,117 +0,0 @@ -#pragma once - -#include "test_server.h" - -#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> - -#include <library/cpp/testing/unittest/registar.h> - -#include <grpc++/client_context.h> -#include <grpc++/create_channel.h> - -namespace NPersQueue::NTests { - -class TPQDataWriter { -public: - TPQDataWriter( - const TString& defaultTopicFullName, const TString& defaultShortTopicName, - const TString& sourceId, TTestServer& server - ) - : FullTopicName(defaultTopicFullName) - , ShortTopicName(defaultShortTopicName) - , SourceId_(sourceId) - , Server(server) - { - InitializeChannel(); - WaitWritePQServiceInitialization(); - } - - void Read(const TString& topic, const TString& clientId, const TString& ticket = "", bool error = false, - bool checkACL = false, bool useBatching = false, bool onlyCreate = false); - - void WaitWritePQServiceInitialization(); - - ui32 InitSession(const TString& sourceId, ui32 pg, bool success, ui32 step = 0); - - ui32 Write(const TString& topic, const TString& data, bool error = false, const TString& ticket = "") { - return WriteImpl(topic, {data}, error, ticket, false); - } - - ui32 WriteBatch(const TString& topic, const TVector<TString>& data, bool error = false, - const TString& ticket = "") { - return WriteImpl(topic, data, error, ticket, true); - } - -private: - ui32 WriteImpl(const TString& topic, const TVector<TString>& data, bool error, const TString& ticket, - bool batch); - - template<typename S> - void Flush(const TVector<TString>& data, S& stream, const TString& ticket, bool batch) { - TWriteRequest request; - TWriteResponse response; - - TVector<ui64> allSeqNo; - if (batch) { - for (const TString& d: data) { - ui64 seqNo = AtomicIncrement(SeqNo_); - allSeqNo.push_back(seqNo); - auto *mutableData = request.MutableDataBatch()->AddData(); - mutableData->SetSeqNo(seqNo); - mutableData->SetData(d); - } - } else { - ui64 seqNo = AtomicIncrement(SeqNo_); - allSeqNo.push_back(seqNo); - request.MutableData()->SetSeqNo(seqNo); - request.MutableData()->SetData(JoinSeq("\n", data)); - } - if (!ticket.empty()) { - request.MutableCredentials()->SetTvmServiceTicket(ticket); - } - - Cerr << "request: " << request << Endl; - if (!stream->Write(request)) { - ythrow yexception() << "write fail"; - } - - UNIT_ASSERT(stream->Read(&response)); - if (batch) { - UNIT_ASSERT_C(response.HasAckBatch(), response); - UNIT_ASSERT_VALUES_EQUAL(data.size(), response.GetAckBatch().AckSize()); - for (size_t i = 0; i < data.size(); ++i) { - const auto& ack = response.GetAckBatch().GetAck(i); - UNIT_ASSERT(!ack.GetAlreadyWritten()); - UNIT_ASSERT(!ack.HasStat()); - UNIT_ASSERT_VALUES_EQUAL(ack.GetSeqNo(), allSeqNo[i]); - } - UNIT_ASSERT(response.GetAckBatch().HasStat()); - } else { - const auto& ack = response.GetAck(); - UNIT_ASSERT(!ack.GetAlreadyWritten()); - UNIT_ASSERT_VALUES_EQUAL(ack.GetSeqNo(), allSeqNo[0]); - UNIT_ASSERT(ack.HasStat()); - } - } - - ui64 ReadCookieFromMetadata(const std::multimap<grpc::string_ref, grpc::string_ref>& meta) const; - - void InitializeChannel(); - -private: - TString FullTopicName; - TString ShortTopicName; - const TString SourceId_; - - TTestServer& Server; - - TAtomic SeqNo_ = 1; - - //! Сетевой канал взаимодействия с proxy-сервером. - std::shared_ptr<grpc::Channel> Channel_; - std::unique_ptr<NKikimrClient::TGRpcServer::Stub> Stub_; - std::unique_ptr<PersQueueService::Stub> StubP_; - - ui64 ProxyCookie = 0; -}; -} // NKikimr::NPersQueueTests diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h deleted file mode 100644 index 9f0712933a8..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h +++ /dev/null @@ -1,323 +0,0 @@ -#pragma once -#include "test_server.h" -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> -#include <ydb/library/persqueue/topic_parser_public/topic_parser.h> -#include <library/cpp/logger/log.h> -#include <util/system/tempfile.h> - -#define TEST_CASE_NAME (this->Name_) - -namespace NPersQueue { -class SDKTestSetup { -protected: - TString TestCaseName; - - THolder<TTempFileHandle> NetDataFile; - THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> DataCenters; - TString LocalDC = "dc1"; - TTestServer Server = TTestServer(false /* don't start */); - - TLog Log = TLog("cerr"); - - TPQLibSettings PQLibSettings; - THolder<TPQLib> PQLib; - -public: - SDKTestSetup(const TString& testCaseName, bool start = true) - : TestCaseName(testCaseName) - { - InitOptions(); - if (start) { - Start(); - } - } - - void InitOptions() { - Log.SetFormatter([testCaseName = TestCaseName](ELogPriority priority, TStringBuf message) { - return TStringBuilder() << TInstant::Now() << " :" << testCaseName << " " << priority << ": " << message << Endl; - }); - Server.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::Max()); - // Default TTestServer value for 'MaxReadCookies' is 10. With this value the tests are flapping with two errors: - // 1. 'got more than 10 unordered cookies to commit 12' - // 2. 'got more than 10 uncommitted reads' - Server.ServerSettings.PQConfig.Clear(); - Server.ServerSettings.PQConfig.SetEnabled(true); - Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1); - Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1); - Server.ServerSettings.PQClusterDiscoveryConfig.SetEnabled(true); - SetNetDataViaFile("::1/128\t" + GetLocalCluster()); - - auto seed = TInstant::Now().MicroSeconds(); - // This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable - Log << TLOG_INFO << "Random seed for debugging is " << seed; - std::srand(seed); - } - - void Start(bool waitInit = true, bool addBrokenDatacenter = !std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED")) { - Server.StartServer(false); - //Server.EnableLogs({NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY}); - Server.AnnoyingClient->InitRoot(); - if (DataCenters.empty()) { - THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> dataCenters; - dataCenters.emplace("dc1", NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true}); - if (addBrokenDatacenter) { - dataCenters.emplace("dc2", NKikimr::NPersQueueTests::TPQTestClusterInfo{"dc2.logbroker.yandex.net", false}); - } - Server.AnnoyingClient->InitDCs(dataCenters); - } else { - Server.AnnoyingClient->InitDCs(DataCenters, LocalDC); - } - Server.AnnoyingClient->InitSourceIds(); - CreateTopic(GetTestTopic(), GetLocalCluster()); - if (waitInit) { - Server.WaitInit(GetTestTopic()); - } - PQLibSettings.DefaultLogger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); - PQLib = MakeHolder<TPQLib>(PQLibSettings); - } - - THolder<TPQLib>& GetPQLib() { - return PQLib; - } - - const TPQLibSettings& GetPQLibSettings() const { - return PQLibSettings; - } - - TString GetTestTopic() const { - return "topic1"; - } - - TString GetTestClient() const { - return "test-reader"; - } - - TString GetTestMessageGroupId() const { - return "test-message-group-id"; - } - - TString GetLocalCluster() const { - return LocalDC; - } - - - NGrpc::TServerOptions& GetGrpcServerOptions() { - return Server.GrpcServerOptions; - } - - void SetNetDataViaFile(const TString& netDataTsv) { - NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); - NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size()); - NetDataFile->FlushData(); - Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); - } - - TProducerSettings GetProducerSettings() const { - TProducerSettings producerSettings; - producerSettings.Topic = GetTestTopic(); - producerSettings.SourceId = GetTestMessageGroupId(); - producerSettings.Server = TServerSetting{"localhost", Server.GrpcPort}; - producerSettings.Server.Database = "/Root"; - return producerSettings; - } - - TConsumerSettings GetConsumerSettings() const { - TConsumerSettings consumerSettings; - consumerSettings.Topics = {GetTestTopic()}; - consumerSettings.ClientId = GetTestClient(); - consumerSettings.Server = TServerSetting{"localhost", Server.GrpcPort}; - consumerSettings.UseV2RetryPolicyInCompatMode = true; - return consumerSettings; - } - - TLog& GetLog() { - return Log; - } - - - template <class TConsumerOrProducer> - void Start(const THolder<TConsumerOrProducer>& obj) { - auto startFuture = obj->Start(); - const auto& initResponse = startFuture.GetValueSync(); - UNIT_ASSERT_C(!initResponse.Response.HasError(), "Failed to start: " << initResponse.Response); - } - - THolder<IConsumer> StartConsumer(const TConsumerSettings& settings) { - THolder<IConsumer> consumer = GetPQLib()->CreateConsumer(settings); - Start(consumer); - return consumer; - } - - THolder<IConsumer> StartConsumer() { - return StartConsumer(GetConsumerSettings()); - } - - THolder<IProducer> StartProducer(const TProducerSettings& settings) { - THolder<IProducer> producer = GetPQLib()->CreateProducer(settings); - Start(producer); - return producer; - } - - THolder<IProducer> StartProducer() { - return StartProducer(GetProducerSettings()); - } - - void WriteToTopic(const TVector<TString>& data, IProducer* producer = nullptr) { - THolder<IProducer> localProducer; - if (!producer) { - localProducer = StartProducer(); - producer = localProducer.Get(); - } - TVector<NThreading::TFuture<TProducerCommitResponse>> resps; - for (const TString& d : data) { - Log << TLOG_INFO << "WriteToTopic: " << d; - resps.push_back(producer->Write(d)); - } - for (NThreading::TFuture<TProducerCommitResponse>& r : resps) { - UNIT_ASSERT_C(!r.GetValueSync().Response.HasError(), r.GetValueSync().Response.GetError()); - } - } - - // Read set of sequences from topic - void ReadFromTopic(const TVector<TVector<TString>>& data, bool commit = true, IConsumer* consumer = nullptr) { - THolder<IConsumer> localConsumer; - if (!consumer) { - localConsumer = StartConsumer(); - consumer = localConsumer.Get(); - } - TVector<size_t> positions(data.size()); // Initialy zeroes. - - int wholeCount = 0; - for (const TVector<TString>& seq : data) { - wholeCount += seq.size(); - } - - TSet<ui64> cookies; - - auto processCommit = [&](const TConsumerMessage& resp) { - Log << TLOG_INFO << "ReadFromTopic. Committed: " << resp.Response.GetCommit(); - for (ui64 cookie : resp.Response.GetCommit().GetCookie()) { - UNIT_ASSERT_VALUES_EQUAL(cookies.erase(cookie), 1); - } - }; - - while (wholeCount > 0) { - auto event = consumer->GetNextMessage(); - const auto& resp = event.GetValueSync(); - UNIT_ASSERT_C(!resp.Response.HasError(), resp.Response); - if (!resp.Response.HasData()) { - if (resp.Response.HasCommit()) { - processCommit(resp); - } - continue; - } - Log << TLOG_INFO << "ReadFromTopic. Data: " << resp.Response.GetData(); - UNIT_ASSERT(cookies.insert(resp.Response.GetData().GetCookie()).second); - for (const auto& batch : resp.Response.GetData().GetMessageBatch()) { - // find proper sequence - const TString& firstData = batch.GetMessage(0).GetData(); - size_t seqIndex = 0; - for (; seqIndex < positions.size(); ++seqIndex) { - if (positions[seqIndex] >= data[seqIndex].size()) { // Already seen. - continue; - } - size_t& seqPos = positions[seqIndex]; - const TString& expectedData = data[seqIndex][seqPos]; - if (expectedData == firstData) { - UNIT_ASSERT(batch.MessageSize() <= data[seqIndex].size() - positions[seqIndex]); - ++seqPos; - --wholeCount; - // Match. - for (size_t msgIndex = 1; msgIndex < batch.MessageSize(); ++msgIndex, ++seqPos, --wholeCount) { - UNIT_ASSERT_STRINGS_EQUAL(batch.GetMessage(msgIndex).GetData(), data[seqIndex][seqPos]); - } - break; - } - } - UNIT_ASSERT_LT_C(seqIndex, positions.size(), resp.Response); - } - - if (commit) { - consumer->Commit({resp.Response.GetData().GetCookie()}); - } - } - while (commit && !cookies.empty()) { - auto event = consumer->GetNextMessage(); - const auto& resp = event.GetValueSync(); - UNIT_ASSERT_C(!resp.Response.HasError(), resp.Response); - if (!resp.Response.HasCommit()) { - continue; - } - processCommit(resp); - } - UNIT_ASSERT_VALUES_EQUAL(wholeCount, 0); - } - - void SetSingleDataCenter(const TString& name = "dc1") { - UNIT_ASSERT( - DataCenters.insert(std::make_pair( - name, - NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true} - )).second - ); - LocalDC = name; - } - - void AddDataCenter(const TString& name, const TString& address, bool enabled = true, bool setSelfAsDc = true) { - if (DataCenters.empty() && setSelfAsDc) { - SetSingleDataCenter(); - } - NKikimr::NPersQueueTests::TPQTestClusterInfo info{ - address, - enabled - }; - UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second); - } - - void AddDataCenter(const TString& name, const SDKTestSetup& cluster, bool enabled = true, bool setSelfAsDc = true) { - AddDataCenter(name, TStringBuilder() << "localhost:" << cluster.Server.GrpcPort, enabled, setSelfAsDc); - } - - void EnableDataCenter(const TString& name) { - auto iter = DataCenters.find(name); - UNIT_ASSERT(iter != DataCenters.end()); - Server.AnnoyingClient->UpdateDcEnabled(name, true); - } - void DisableDataCenter(const TString& name) { - auto iter = DataCenters.find(name); - UNIT_ASSERT(iter != DataCenters.end()); - Server.AnnoyingClient->UpdateDcEnabled(name, false); - } - - void ShutdownGRpc() { - Server.ShutdownGRpc(); - } - - void EnableGRpc() { - Server.EnableGRpc(); - Server.WaitInit(GetTestTopic()); - } - - void KickTablets() { - for (ui32 i = 0; i < Server.CleverServer->StaticNodes(); i++) { - Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false); - Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i); - } - } - void AllowTablets() { - for (ui32 i = 0; i < Server.CleverServer->StaticNodes(); i++) { - Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true); - } - } - - void CreateTopic(const TString& topic, const TString& cluster, size_t partitionsCount = 1) { - Server.AnnoyingClient->CreateTopicNoLegacy(BuildFullTopicName(topic, cluster), partitionsCount); - } - - void KillPqrb(const TString& topic, const TString& cluster) { - auto describeResult = Server.AnnoyingClient->Ls(TStringBuilder() << "/Root/PQ/" << BuildFullTopicName(topic, cluster)); - UNIT_ASSERT_C(describeResult->Record.GetPathDescription().HasPersQueueGroup(), describeResult->Record); - Server.AnnoyingClient->KillTablet(*Server.CleverServer, describeResult->Record.GetPathDescription().GetPersQueueGroup().GetBalancerTabletID()); - } -}; -} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp deleted file mode 100644 index f5d0b63cb36..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include "test_pqlib.h" - -namespace NPersQueue::NTests { - - std::tuple<THolder<IProducer>, TProducerCreateResponse> TTestPQLib::CreateProducer( - const TProducerSettings& settings, bool deprecated - ) { - auto producer = PQLib->CreateProducer(settings, Logger, deprecated); - TProducerCreateResponse response = producer->Start().GetValueSync(); - return std::tuple<THolder<IProducer>, TProducerCreateResponse>(std::move(producer), response); - } - - std::tuple<THolder<IProducer>, TProducerCreateResponse> TTestPQLib::CreateProducer( - const TString& topic, - const TString& sourceId, - std::optional<ui32> partitionGroup, - std::optional<NPersQueueCommon::ECodec> codec, - std::optional<bool> reconnectOnFailure, - bool deprecated - ) { - TProducerSettings s; - s.Server = TServerSetting{"localhost", Server.GrpcPort}; - s.Topic = topic; - s.SourceId = sourceId; - if (partitionGroup) { - s.PartitionGroup = partitionGroup.value(); - } - if (codec) { - s.Codec = codec.value(); - } - if (reconnectOnFailure) { - s.ReconnectOnFailure = reconnectOnFailure.value(); - } - return CreateProducer(s, deprecated); - } - - std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateConsumer( - const TConsumerSettings& settings - ) { - auto consumer = PQLib->CreateConsumer(settings, Logger); - TConsumerCreateResponse response = consumer->Start().GetValueSync(); - return std::tuple<THolder<IConsumer>, TConsumerCreateResponse>(std::move(consumer), response); - } - - std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateDeprecatedConsumer( - const TConsumerSettings& settings - ) { - auto consumer = PQLib->CreateConsumer(settings, Logger, true); - TConsumerCreateResponse response = consumer->Start().GetValueSync(); - return std::tuple<THolder<IConsumer>, TConsumerCreateResponse>(std::move(consumer), response); - } - - TConsumerSettings TTestPQLib::MakeSettings( - const TVector<TString>& topics, - const TString& consumerName - ) { - TConsumerSettings s; - s.Server = TServerSetting{"localhost", Server.GrpcPort}; - s.ClientId = consumerName; - s.Topics = topics; - return s; - } - - std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateConsumer( - const TVector<TString>& topics, - const TString& consumerName, - std::optional<ui32> maxCount, - std::optional<bool> useLockSession, - std::optional<bool> readMirroredPartitions, - std::optional<bool> unpack, - std::optional<ui32> maxInflyRequests, - std::optional<ui32> maxMemoryUsage - ) { - auto s = MakeSettings(topics, consumerName); - if (maxCount) { - s.MaxCount = maxCount.value(); - } - if (unpack) { - s.Unpack = unpack.value(); - } - if (readMirroredPartitions) { - s.ReadMirroredPartitions = readMirroredPartitions.value(); - } - if (maxInflyRequests) { - s.MaxInflyRequests = maxInflyRequests.value(); - } - if (maxMemoryUsage) { - s.MaxMemoryUsage = maxMemoryUsage.value(); - } - if (useLockSession) { - s.UseLockSession = useLockSession.value(); - } - return CreateConsumer(s); - } -} // namespace NPersQueue::NTests
\ No newline at end of file diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h deleted file mode 100644 index 8e6fb6832dc..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once -#include "test_server.h" - -namespace NPersQueue::NTests { - -class TTestPQLib { -public: - TTestPQLib(TTestServer& server) - : Logger(new TCerrLogger(DEBUG_LOG_LEVEL)) - , Server(server) - { - TPQLibSettings settings; - settings.DefaultLogger = Logger; - PQLib = MakeHolder<TPQLib>(settings); - } - - std::tuple<THolder<IProducer>, TProducerCreateResponse> CreateProducer( - const TProducerSettings& settings, bool deprecated - ); - - std::tuple<THolder<IProducer>, TProducerCreateResponse> CreateProducer( - const TString& topic, const TString& sourceId, std::optional<ui32> partitionGroup = {}, - std::optional<NPersQueueCommon::ECodec> codec = ECodec::RAW, std::optional<bool> reconnectOnFailure = {}, - bool deprecated = false); - - std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateConsumer(const TConsumerSettings& settings); - - std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateDeprecatedConsumer(const TConsumerSettings& settings); - - TConsumerSettings MakeSettings(const TVector<TString>& topics, const TString& consumerName = "user"); - - std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateConsumer( - const TVector<TString>& topics, const TString& consumerName = "user", std::optional<ui32> maxCount = {}, - std::optional<bool> useLockSession = {}, std::optional<bool> readMirroredPartitions = {}, - std::optional<bool> unpack = {}, std::optional<ui32> maxInflyRequests = {}, - std::optional<ui32> maxMemoryUsage = {}); - -private: - TIntrusivePtr<TCerrLogger> Logger; - TTestServer& Server; - THolder<TPQLib> PQLib; -}; -} // namespace NPersQueue::NTests
\ No newline at end of file diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp deleted file mode 100644 index 6cd1e8094a0..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "test_server.h" - -#include <ydb/core/client/server/msgbus_server_pq_metacache.h> -#include <kikimr/yndx/grpc_services/persqueue/persqueue.h> -#include <kikimr/yndx/persqueue/msgbus_server/read_session_info.h> - -namespace NPersQueue { - -const TVector<NKikimrServices::EServiceKikimr> TTestServer::LOGGED_SERVICES = { - NKikimrServices::PERSQUEUE, - NKikimrServices::PQ_METACACHE, - NKikimrServices::PQ_READ_PROXY, - NKikimrServices::PQ_WRITE_PROXY, - NKikimrServices::PQ_MIRRORER, -}; - -void TTestServer::PatchServerSettings() { - ServerSettings.RegisterGrpcService<NKikimr::NGRpcService::TGRpcPersQueueService>( - "pq", - NKikimr::NMsgBusProxy::CreatePersQueueMetaCacheV2Id() - ); - ServerSettings.SetPersQueueGetReadSessionsInfoWorkerFactory( - std::make_shared<NKikimr::NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerWithPQv0Factory>() - ); -} - -void TTestServer::StartIfNeeded(bool start) { - if (start) { - StartServer(); - } -} - - - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h deleted file mode 100644 index 090d9aa47f4..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h +++ /dev/null @@ -1,129 +0,0 @@ -#pragma once -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -#include <ydb/core/testlib/test_pq_client.h> -#include <library/cpp/grpc/server/grpc_server.h> - -#include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/testing/unittest/tests_data.h> - -#include <util/system/tempfile.h> - -namespace NPersQueue { - -static constexpr int DEBUG_LOG_LEVEL = 7; - -class TTestServer { -public: - TTestServer(bool start = true, TMaybe<TSimpleSharedPtr<TPortManager>> portManager = Nothing()) - : PortManager(portManager.GetOrElse(MakeSimpleShared<TPortManager>())) - , Port(PortManager->GetPort(2134)) - , GrpcPort(PortManager->GetPort(2135)) - , ServerSettings(NKikimr::NPersQueueTests::PQSettings(Port).SetGrpcPort(GrpcPort)) - , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) - , PQLibSettings(TPQLibSettings{ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) }) - , PQLib(new TPQLib(PQLibSettings)) - { - PatchServerSettings(); - StartIfNeeded(start); - } - - TTestServer(const NKikimr::Tests::TServerSettings& settings, bool start = true) - : PortManager(MakeSimpleShared<TPortManager>()) - , Port(PortManager->GetPort(2134)) - , GrpcPort(PortManager->GetPort(2135)) - , ServerSettings(settings) - , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) - , PQLibSettings(TPQLibSettings{ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) }) - , PQLib(new TPQLib(PQLibSettings)) - { - ServerSettings.Port = Port; - ServerSettings.SetGrpcPort(GrpcPort); - PatchServerSettings(); - StartIfNeeded(start); - } - - void StartServer(bool doClientInit = true) { - PrepareNetDataFile(); - CleverServer = MakeHolder<NKikimr::Tests::TServer>(ServerSettings); - CleverServer->EnableGRpc(GrpcServerOptions); - AnnoyingClient = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(ServerSettings, GrpcPort); - EnableLogs(LOGGED_SERVICES); - if (doClientInit) { - AnnoyingClient->FullInit(); - } - } - - void ShutdownGRpc() { - CleverServer->ShutdownGRpc(); - } - - void EnableGRpc() { - CleverServer->EnableGRpc(GrpcServerOptions); - } - - void ShutdownServer() { - CleverServer = nullptr; - } - - void RestartServer() { - ShutdownServer(); - StartServer(); - } - - void EnableLogs(const TVector<NKikimrServices::EServiceKikimr> services, - NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) { - Y_VERIFY(CleverServer != nullptr, "Start server before enabling logs"); - for (auto s : services) { - CleverServer->GetRuntime()->SetLogPriority(s, prio); - } - } - - void WaitInit(const TString& topic) { - TProducerSettings s; - s.Topic = topic; - s.Server = TServerSetting{"localhost", GrpcPort}; - s.SourceId = "src"; - - while (PQLib->CreateProducer(s, {}, false)->Start().GetValueSync().Response.HasError()) { - Sleep(TDuration::MilliSeconds(200)); - } - } - - bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") { - if (NetDataFile) - return false; - NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); - NetDataFile->Write(content.Data(), content.Size()); - NetDataFile->FlushData(); - ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); - return true; - } - - void UpdateDC(const TString& name, bool local, bool enabled) { - AnnoyingClient->UpdateDC(name, local, enabled); - } - -private: - void PatchServerSettings(); - void StartIfNeeded(bool start); - -public: - TSimpleSharedPtr<TPortManager> PortManager; - ui16 Port; - ui16 GrpcPort; - - THolder<NKikimr::Tests::TServer> CleverServer; - NKikimr::Tests::TServerSettings ServerSettings; - NGrpc::TServerOptions GrpcServerOptions; - THolder<TTempFileHandle> NetDataFile; - - THolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient> AnnoyingClient; - - TPQLibSettings PQLibSettings; - THolder<TPQLib> PQLib; - - static const TVector<NKikimrServices::EServiceKikimr> LOGGED_SERVICES; -}; - -} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h deleted file mode 100644 index 13b4da27727..00000000000 --- a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h +++ /dev/null @@ -1,101 +0,0 @@ -#pragma once -#include <util/generic/ptr.h> -#include <util/generic/size_literals.h> -#include <library/cpp/threading/chunk_queue/queue.h> -#include <util/generic/overloaded.h> -#include <library/cpp/testing/unittest/registar.h> - -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> -#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> - -namespace NPersQueue { - -using namespace NThreading; -using namespace NYdb::NPersQueue; -using namespace NKikimr; -using namespace NKikimr::NPersQueueTests; -//using namespace NPersQueue::V1; - -template <class TPQLibObject> -void DestroyAndWait(THolder<TPQLibObject>& object) { - if (object) { - auto isDead = object->IsDead(); - object = nullptr; - isDead.GetValueSync(); - } -} - -inline bool GrpcV1EnabledByDefault() { - static const bool enabled = std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED"); - return enabled; -} - -class TCallbackCredentialsProvider : public ICredentialsProvider { - std::function<void(NPersQueue::TCredentials*)> Callback; -public: - TCallbackCredentialsProvider(std::function<void(NPersQueue::TCredentials*)> callback) - : Callback(std::move(callback)) - {} - - void FillAuthInfo(NPersQueue::TCredentials* authInfo) const { - Callback(authInfo); - } -}; - -struct TWriteResult { - bool Ok = false; - // No acknowledgement is expected from a writer under test - bool NoWait = false; - TString ResponseDebugString = TString(); -}; - -struct TAcknowledgableMessage { - TString Value; - ui64 SequenceNumber; - TInstant CreatedAt; - TPromise<TWriteResult> AckPromise; -}; - -class IClientEventLoop { -protected: - std::atomic_bool MayStop; - std::atomic_bool MustStop; - bool Stopped = false; - std::unique_ptr<TThread> Thread; - TLog Log; - -public: - IClientEventLoop() - : MayStop() - , MustStop() - , MessageBuffer() - {} - - void AllowStop() { - MayStop = true; - } - - void WaitForStop() { - if (!Stopped) { - Log << TLOG_INFO << "Wait for writer to die on itself"; - Thread->Join(); - Log << TLOG_INFO << "Client write event loop stopped"; - } - Stopped = true; - } - - virtual ~IClientEventLoop() { - MustStop = true; - if (!Stopped) { - Log << TLOG_INFO << "Wait for client write event loop to stop"; - Thread->Join(); - Log << TLOG_INFO << "Client write event loop stopped"; - } - Stopped = true; - } - - TManyOneQueue<TAcknowledgableMessage> MessageBuffer; - -}; - -} // namespace NPersQueue |
