diff options
author | komels <komels@yandex-team.ru> | 2022-04-14 13:10:53 +0300 |
---|---|---|
committer | komels <komels@yandex-team.ru> | 2022-04-14 13:10:53 +0300 |
commit | 21c9b0e6b039e9765eb414c406c2b86e8cea6850 (patch) | |
tree | f40ebc18ff8958dfbd189954ad024043ca983ea5 /kikimr/persqueue/sdk/deprecated | |
parent | 9a4effa852abe489707139c2b260dccc6f4f9aa9 (diff) | |
download | ydb-21c9b0e6b039e9765eb414c406c2b86e8cea6850.tar.gz |
Final part on compatibility layer: LOGBROKER-7215
ref:777c67aadbf705d19034a09a792b2df61ba53697
Diffstat (limited to 'kikimr/persqueue/sdk/deprecated')
89 files changed, 14423 insertions, 0 deletions
diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt b/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt new file mode 100644 index 0000000000..44cb2496ed --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/CMakeLists.txt @@ -0,0 +1,60 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(deprecated-cpp-v2) +target_link_libraries(deprecated-cpp-v2 PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-grpc + contrib-libs-protobuf + cpp-client-iam + cpp-client-ydb_persqueue + api-grpc-yndx + api-protos-yndx + yndx-persqueue-read_batch_converter + library-cpp-blockcodecs + cpp-containers-disjoint_interval_tree + cpp-containers-intrusive_rb_tree + cpp-grpc-common + cpp-http-simple + library-cpp-json + library-cpp-logger + cpp-streams-lzop + cpp-streams-zstd + cpp-string_utils-quote + cpp-threading-future + cpp-tvmauth-client + library-persqueue-topic_parser_public + cpp-client-resources +) +target_sources(deprecated-cpp-v2 PRIVATE + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp +) diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h new file mode 100644 index 0000000000..623a293dc3 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_interface.h @@ -0,0 +1,60 @@ +#pragma once +#include "fake_actor.h" + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/actorid.h> +#include <library/cpp/actors/core/actorsystem.h> +#include <library/cpp/actors/core/events.h> + +namespace NPersQueue { + +struct IActorInterface { + virtual ~IActorInterface() = default; + + // Sender of all messages from PQLib objects. + virtual NActors::TActorId GetActorID() const noexcept = 0; +}; + +class TActorHolder : public IActorInterface +{ +protected: + TActorHolder(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID) + : ActorSystem(actorSystem) + , ParentActorID(parentActorID) + { + ActorID = ActorSystem->Register(new TFakeActor()); + } + + ~TActorHolder() { + ActorSystem->Send(new NActors::IEventHandle(ActorID, ActorID, new NActors::TEvents::TEvPoisonPill())); + } + + template <class TResponseEvent> + void Subscribe(ui64 requestId, NThreading::TFuture<typename TResponseEvent::TResponseType>&& future) { + auto handler = [ + requestId = requestId, + actorSystem = ActorSystem, + actorID = ActorID, + parentActorID = ParentActorID + ](const NThreading::TFuture<typename TResponseEvent::TResponseType>& future) { + actorSystem->Send(new NActors::IEventHandle( + parentActorID, + actorID, + new TResponseEvent(typename TResponseEvent::TResponseType(future.GetValue()), requestId) + )); + }; + future.Subscribe(handler); + } + +public: + NActors::TActorId GetActorID() const noexcept override { + return ActorID; + } + +protected: + NActors::TActorSystem* ActorSystem; + NActors::TActorId ParentActorID; + NActors::TActorId ActorID; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h new file mode 100644 index 0000000000..ce2f7682c1 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers.h @@ -0,0 +1,108 @@ +#pragma once +#include "actor_interface.h" +#include "responses.h" + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> + +#include <util/datetime/base.h> + +namespace NPersQueue { + +template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId> +class TProducerActorWrapper: public TActorHolder { +public: + using TObjectIsDeadEvent = TPQLibResponseEvent<TError, TObjectIsDeadEventId>; + using TCreateResponseEvent = TPQLibResponseEvent<TProducerCreateResponse, TCreateResponseEventId>; + using TCommitResponseEvent = TPQLibResponseEvent<TProducerCommitResponse, TCommitResponseEventId>; + +public: + TProducerActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IProducer> producer) + : TActorHolder(actorSystem, parentActorID) + , Producer(std::move(producer)) + { + } + + void Start(TInstant deadline, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { + Subscribe<TCreateResponseEvent>(requestId, Producer->Start(deadline)); + Subscribe<TObjectIsDeadEvent>(isDeadRequestId, Producer->IsDead()); + } + + void Start(TDuration timeout, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { + Start(TInstant::Now() + timeout, requestId, isDeadRequestId); + } + + void Write(TProducerSeqNo seqNo, TData data, ui64 requestId = 0) noexcept { + Subscribe<TCommitResponseEvent>(requestId, Producer->Write(seqNo, std::move(data))); + } + + void Write(TData data, ui64 requestId = 0) noexcept { + Subscribe<TCommitResponseEvent>(requestId, Producer->Write(std::move(data))); + } + +private: + THolder<IProducer> Producer; +}; + +template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TGetMessageEventId> +class TConsumerActorWrapper: public TActorHolder { +public: + using TObjectIsDeadEvent = TPQLibResponseEvent<TError, TObjectIsDeadEventId>; + using TCreateResponseEvent = TPQLibResponseEvent<TConsumerCreateResponse, TCreateResponseEventId>; + using TGetMessageEvent = TPQLibResponseEvent<TConsumerMessage, TGetMessageEventId>; + +public: + TConsumerActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IConsumer> consumer) + : TActorHolder(actorSystem, parentActorID) + , Consumer(std::move(consumer)) + { + } + + void Start(TInstant deadline, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { + Subscribe<TCreateResponseEvent>(requestId, Consumer->Start(deadline)); + Subscribe<TObjectIsDeadEvent>(isDeadRequestId, Consumer->IsDead()); + } + + void Start(TDuration timeout, ui64 requestId = 0, ui64 isDeadRequestId = 0) noexcept { + Start(TInstant::Now() + timeout, requestId, isDeadRequestId); + } + + void GetNextMessage(ui64 requestId = 0) noexcept { + Subscribe<TGetMessageEvent>(requestId, Consumer->GetNextMessage()); + } + + void Commit(const TVector<ui64>& cookies) noexcept { + Consumer->Commit(cookies); // no future in PQLib API + } + + void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { + Consumer->RequestPartitionStatus(topic, partition, generation); // no future in PQLib API + } + +private: + THolder<IConsumer> Consumer; +}; + +template <ui32 TOriginDataEventId> +class TProcessorActorWrapper: public TActorHolder { +public: + using TOriginDataEvent = TPQLibResponseEvent<TOriginData, TOriginDataEventId>; + +public: + TProcessorActorWrapper(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, THolder<IProcessor> processor) + : TActorHolder(actorSystem, parentActorID) + , Processor(std::move(processor)) + { + } + + void GetNextData(ui64 requestId = 0) noexcept { + Subscribe<TOriginDataEvent>(requestId, Processor->GetNextData()); + } + +private: + THolder<IProcessor> Processor; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp new file mode 100644 index 0000000000..1e08597ec3 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/actor_wrappers_ut.cpp @@ -0,0 +1,119 @@ +#include "actor_wrappers.h" +#include "responses.h" + +#include <ydb/core/testlib/basics/runtime.h> +#include <ydb/core/testlib/basics/appdata.h> + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/testing/gmock_in_unittest/gmock.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/thread.h> + +using namespace testing; + +namespace NPersQueue { + +namespace TEvPQLibTests { + enum EEv { + EvPQLibObjectIsDead = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), + EvPQLibProducerCreateResponse, + EvPQLibProducerCommitResponse, + EvEnd + }; +} // TEvPQLibTests + +class TMockProducer: public IProducer { +public: + NThreading::TFuture<TProducerCreateResponse> Start(TInstant) noexcept override { + return NThreading::MakeFuture<TProducerCreateResponse>(MockStart()); + } + + NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo, TData data) noexcept override { + return Write(std::move(data)); + } + + NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept override { + return NThreading::MakeFuture<TProducerCommitResponse>(MockWrite(data)); + } + + NThreading::TFuture<TError> IsDead() noexcept override { + return MockIsDead(); + } + + MOCK_METHOD(TProducerCreateResponse, MockStart, (), ()); + MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ()); + MOCK_METHOD(TProducerCommitResponse, MockWrite, (TData), ()); +}; + +Y_UNIT_TEST_SUITE(TProducerActorWrapperTest) { + Y_UNIT_TEST(PassesEvents) { + NActors::TTestActorRuntime runtime; + runtime.Initialize(NKikimr::TAppPrepare().Unwrap()); + auto edgeActorId = runtime.AllocateEdgeActor(); + THolder<TMockProducer> producerHolder = MakeHolder<TMockProducer>(); + TMockProducer& producer = *producerHolder; + + // expectations + TWriteResponse startResponse; + startResponse.MutableInit()->SetMaxSeqNo(42); + EXPECT_CALL(producer, MockStart()) + .WillOnce(Return(TProducerCreateResponse(std::move(startResponse)))); + + NThreading::TPromise<TError> deadPromise = NThreading::NewPromise<TError>(); + EXPECT_CALL(producer, MockIsDead()) + .WillOnce(Return(deadPromise.GetFuture())); + + TWriteResponse writeResponse; + writeResponse.MutableAck()->SetSeqNo(100); + TData data("data"); + EXPECT_CALL(producer, MockWrite(data)) + .WillOnce(Return(TProducerCommitResponse(100, data, std::move(writeResponse)))); + + // wrapper creation + using TProducerType = TProducerActorWrapper< + TEvPQLibTests::EvPQLibObjectIsDead, + TEvPQLibTests::EvPQLibProducerCreateResponse, + TEvPQLibTests::EvPQLibProducerCommitResponse + >; + + THolder<TProducerType> wrapper = MakeHolder<TProducerType>(runtime.GetAnyNodeActorSystem(), edgeActorId, std::move(producerHolder)); + + // checks + auto actorId = wrapper->GetActorID(); + UNIT_ASSERT(actorId); + + { + wrapper->Start(TInstant::Max()); + TAutoPtr<NActors::IEventHandle> handle; + auto* startEvent = runtime.GrabEdgeEvent<TProducerType::TCreateResponseEvent>(handle); + UNIT_ASSERT(startEvent != nullptr); + UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId); + UNIT_ASSERT_VALUES_EQUAL(startEvent->Response.Response.GetInit().GetMaxSeqNo(), 42); + } + + { + wrapper->Write(data); + TAutoPtr<NActors::IEventHandle> handle; + auto* writeEvent = runtime.GrabEdgeEvent<TProducerType::TCommitResponseEvent>(handle); + UNIT_ASSERT(writeEvent != nullptr); + UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId); + UNIT_ASSERT_VALUES_EQUAL(writeEvent->Response.Response.GetAck().GetSeqNo(), 100); + UNIT_ASSERT_EQUAL(writeEvent->Response.Data, data); + UNIT_ASSERT_VALUES_EQUAL(writeEvent->Response.SeqNo, 100); + } + + { + TError err; + err.SetDescription("trololo"); + deadPromise.SetValue(err); + + TAutoPtr<NActors::IEventHandle> handle; + auto* deadEvent = runtime.GrabEdgeEvent<TProducerType::TObjectIsDeadEvent>(handle); + UNIT_ASSERT(deadEvent != nullptr); + UNIT_ASSERT_VALUES_EQUAL(handle->Sender, actorId); + UNIT_ASSERT_STRINGS_EQUAL(deadEvent->Response.GetDescription(), "trololo"); + } + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h new file mode 100644 index 0000000000..630e8a35f8 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/fake_actor.h @@ -0,0 +1,24 @@ +#pragma once +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/hfunc.h> + +namespace NPersQueue { + +// Fake object to hold actor id +class TFakeActor: public NActors::TActor<TFakeActor> { +public: + TFakeActor() + : TActor<TFakeActor>(&TFakeActor::StateFunc) + { + } + +private: + STFUNC(StateFunc) { + switch (ev->GetTypeRewrite()) { + CFunc(NActors::TEvents::TSystem::PoisonPill, Die); + } + } +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h new file mode 100644 index 0000000000..4ad5fb0d48 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/logger.h @@ -0,0 +1,48 @@ +#pragma once +#include "responses.h" +#include "actor_interface.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h> + +namespace NPersQueue { + +template <ui32 EventTypeId> +class TActorLogger + : public TActorHolder + , public ILogger +{ +public: + using TLogEvent = TEvPQLibLog<EventTypeId>; + +public: + explicit TActorLogger(NActors::TActorSystem* actorSystem, const NActors::TActorId& parentActorID, int level) + : TActorHolder(actorSystem, parentActorID) + , Level(level) + { + } + + ~TActorLogger() override = default; + + void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) override { + if (!IsEnabled(level)) { + return; + } + ActorSystem->Send(ParentActorID, new TLogEvent( + msg, + sourceId, + sessionId, + static_cast<NActors::NLog::EPriority>(level) + )); + } + + bool IsEnabled(int level) const override { + return level <= Level; + } + +private: + int Level; +}; + +} + + + diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp new file mode 100644 index 0000000000..c663c3ac04 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.cpp @@ -0,0 +1,22 @@ +#include "actor_wrappers.h" +#include "persqueue.h" + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h> + +namespace NPersQueue { + +TPQLibActorsWrapper::TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, const TPQLibSettings& settings) + : PQLib(new TPQLib(settings)) + , ActorSystem(actorSystem) +{ +} + +TPQLibActorsWrapper::TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, std::shared_ptr<TPQLib> pqLib) + : PQLib(std::move(pqLib)) + , ActorSystem(actorSystem) +{ +} + +TPQLibActorsWrapper::~TPQLibActorsWrapper() = default; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h new file mode 100644 index 0000000000..7a11416f4f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/persqueue.h @@ -0,0 +1,87 @@ +#pragma once +#include "actor_wrappers.h" + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +#include <library/cpp/actors/core/actorid.h> + +#include <util/generic/noncopyable.h> +#include <util/generic/ptr.h> + +namespace NActors { +class TActorSystem; +} // namespace NActors + +namespace NPersQueue { + +class TPQLibActorsWrapper : public TNonCopyable { +public: + explicit TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, const TPQLibSettings& settings = TPQLibSettings()); + TPQLibActorsWrapper(NActors::TActorSystem* actorSystem, std::shared_ptr<TPQLib> pqLib); + ~TPQLibActorsWrapper(); + + // Producers creation + template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId> + THolder<TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>> CreateProducer( + const NActors::TActorId& parentActorID, + const TProducerSettings& settings, + TIntrusivePtr<ILogger> logger = nullptr, + bool deprecated = false + ) { + return new TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>( + ActorSystem, + parentActorID, + PQLib->CreateProducer(settings, std::move(logger), deprecated) + ); + } + + template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TCommitResponseEventId> + THolder<TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>> CreateMultiClusterProducer( + const NActors::TActorId& parentActorID, + const TMultiClusterProducerSettings& settings, + TIntrusivePtr<ILogger> logger = nullptr, + bool deprecated = false + ) { + return new TProducerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TCommitResponseEventId>( + ActorSystem, + parentActorID, + PQLib->CreateMultiClusterProducer(settings, std::move(logger), deprecated) + ); + } + + // Consumers creation + template <ui32 TObjectIsDeadEventId, ui32 TCreateResponseEventId, ui32 TGetMessageEventId> + THolder<TConsumerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TGetMessageEventId>> CreateConsumer( + const NActors::TActorId& parentActorID, + const TConsumerSettings& settings, + TIntrusivePtr<ILogger> logger = nullptr, + bool deprecated = false + ) { + return MakeHolder<TConsumerActorWrapper<TObjectIsDeadEventId, TCreateResponseEventId, TGetMessageEventId>>( + ActorSystem, + parentActorID, + PQLib->CreateConsumer(settings, std::move(logger),deprecated) + ); + } + + // Processors creation + template <ui32 TOriginDataEventId> + THolder<TProcessorActorWrapper<TOriginDataEventId>> CreateProcessor( + const NActors::TActorId& parentActorID, + const TProcessorSettings& settings, + TIntrusivePtr<ILogger> logger = nullptr, + bool deprecated = false + ) { + return new TProcessorActorWrapper<TOriginDataEventId>( + ActorSystem, + parentActorID, + PQLib->CreateProcessor(settings, std::move(logger), deprecated) + ); + } + +private: + std::shared_ptr<TPQLib> PQLib; + NActors::TActorSystem* ActorSystem; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h new file mode 100644 index 0000000000..86e732d87c --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/actors/responses.h @@ -0,0 +1,39 @@ +#pragma once +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> + +#include <library/cpp/actors/core/events.h> +#include <library/cpp/actors/core/event_local.h> + +namespace NPersQueue { + +template <class TResponseType_, ui32 EventTypeId> +struct TPQLibResponseEvent: public NActors::TEventLocal<TPQLibResponseEvent<TResponseType_, EventTypeId>, EventTypeId> { + using TResponseType = TResponseType_; + + explicit TPQLibResponseEvent(TResponseType&& response, ui64 requestId = 0) + : Response(std::move(response)) + , RequestId(requestId) + { + } + + TResponseType Response; + ui64 RequestId; +}; + +template <ui32 EventTypeId> +struct TEvPQLibLog : NActors::TEventLocal<TEvPQLibLog<EventTypeId>, EventTypeId> { + TEvPQLibLog(const TString& msg, const TString& sourceId, const TString& sessionId, NActors::NLog::EPriority level) + : Message(msg) + , SourceId(sourceId) + , SessionId(sessionId) + , Level(level) + {} + + TString Message; + TString SourceId; + TString SessionId; + NActors::NLog::EPriority Level; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp new file mode 100644 index 0000000000..b4d8431c8a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/compatibility_ut/compatibility_ut.cpp @@ -0,0 +1,113 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + +namespace NPersQueue { +Y_UNIT_TEST_SUITE(TCompatibilityTest) { + void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { + const TProducerCommitResponse& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); + } + + void AssertWriteFailed(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { + const TProducerCommitResponse& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kError, "Msg: " << resp.Response); + } + + void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); + } + + void AssertReadContinuous(const TReadResponse::TData::TMessageBatch& batch, ui64 startSeqNo) { + for (ui32 i = 0; i < batch.MessageSize(); ++i) { + ui64 actualSeqNo = batch.GetMessage(i).GetMeta().GetSeqNo(); + UNIT_ASSERT_EQUAL_C(actualSeqNo, startSeqNo + i, "Wrong seqNo: " << actualSeqNo << ", expected: " << startSeqNo + i); + } + } + + void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response); + } + + void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response); + } + + void AssertLock(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kLock, "Msg: " << resp.Response); + } + + TProducerSettings MakeProducerSettings(const TTestServer& testServer) { + TProducerSettings producerSettings; + producerSettings.ReconnectOnFailure = true; + producerSettings.Topic = "topic1"; + producerSettings.SourceId = "123"; + producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + producerSettings.Codec = ECodec::LZOP; + producerSettings.ReconnectionDelay = TDuration::MilliSeconds(10); + return producerSettings; + } + + TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { + TConsumerSettings consumerSettings; + consumerSettings.ClientId = "user"; + consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + consumerSettings.Topics.push_back("topic1"); + return consumerSettings; + } + + Y_UNIT_TEST(ContinuesOperationsAfterPQLibDeath) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(); + + const size_t partitions = 1; + testServer.AnnoyingClient->FullInit(); + testServer.AnnoyingClient->CreateTopic("rt3.dc1--topic1", partitions); + + testServer.WaitInit("topic1"); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLibSettings pqLibSettings; + pqLibSettings.DefaultLogger = logger; + THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); + + auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, true); + UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); + auto isProducerDead = producer->IsDead(); + UNIT_ASSERT(!isProducerDead.HasValue()); + + auto consumer = PQLib->CreateConsumer(MakeConsumerSettings(testServer), logger, true); + + PQLib = nullptr; + + UNIT_ASSERT(!consumer->Start().GetValueSync().Response.HasError()); + auto isConsumerDead = consumer->IsDead(); + UNIT_ASSERT(!isConsumerDead.HasValue()); + + auto write1 = producer->Write(1, TString("blob1")); + auto write2 = producer->Write(2, TString("blob2")); + + UNIT_ASSERT(!write1.GetValueSync().Response.HasError()); + UNIT_ASSERT(!write2.GetValueSync().Response.HasError()); + + UNIT_ASSERT(!consumer->GetNextMessage().GetValueSync().Response.HasError()); + + producer = nullptr; + consumer = nullptr; + + isProducerDead.GetValueSync(); + isConsumerDead.GetValueSync(); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h new file mode 100644 index 0000000000..48df7b0cdd --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h @@ -0,0 +1,63 @@ +#pragma once + +#include "logger.h" + +#include <kikimr/yndx/api/protos/persqueue.pb.h> + +#include <library/cpp/tvmauth/client/facade.h> + +namespace NPersQueue { + +const TString& GetTvmPqServiceName(); + +class ICredentialsProvider { +public: + virtual ~ICredentialsProvider() = default; + virtual void FillAuthInfo(NPersQueueCommon::TCredentials* authInfo) const = 0; +}; + + +class ITVMCredentialsForwarder : public ICredentialsProvider { +public: + virtual ~ITVMCredentialsForwarder() = default; + virtual void SetTVMServiceTicket(const TString& ticket) = 0; +}; + +std::shared_ptr<ICredentialsProvider> CreateInsecureCredentialsProvider(); + +/* + * Builder for direct usage of TVM. For Qloud environment use CreateTVMQloudCredentialsProvider + */ +std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, TIntrusivePtr<ILogger> logger = nullptr); +std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, const TString& dstAlias = GetTvmPqServiceName()); + +/* + * forward TVM service ticket with ITMVCredentialsProvider method SetTVMServiceTicket ; threadsafe + */ +std::shared_ptr<ITVMCredentialsForwarder> CreateTVMCredentialsForwarder(); + +/* + * tvmClient settings must contain LogBroker client id under an alias + */ +std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, TIntrusivePtr<ILogger> logger = nullptr, const TString& dstAlias = GetTvmPqServiceName()); + +std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, TIntrusivePtr<ILogger> logger = nullptr, const TDuration refreshPeriod = TDuration::Minutes(10), ui32 port = 1); +std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, TIntrusivePtr<ILogger> logger = nullptr, const TDuration refreshPeriod = TDuration::Minutes(10), ui32 port = 1); +std::shared_ptr<ICredentialsProvider> CreateOAuthCredentialsProvider(const TString& token); + +/* + * Provider that takes IAM ticket from metadata service. + */ +std::shared_ptr<ICredentialsProvider> CreateIAMCredentialsForwarder(TIntrusivePtr<ILogger> logger); + +/* + * Provider that makes IAM ticket from JWT token. + */ +std::shared_ptr<ICredentialsProvider> CreateIAMJwtFileCredentialsForwarder(const TString& jwtKeyFilename, TIntrusivePtr<ILogger> logger, + const TString& endpoint = "iam.api.cloud.yandex.net", const TDuration& refreshPeriod = TDuration::Hours(1), + const TDuration& requestTimeout = TDuration::Seconds(10)); + +std::shared_ptr<ICredentialsProvider> CreateIAMJwtParamsCredentialsForwarder(const TString& jwtParams, TIntrusivePtr<ILogger> logger, + const TString& endpoint = "iam.api.cloud.yandex.net", const TDuration& refreshPeriod = TDuration::Hours(1), + const TDuration& requestTimeout = TDuration::Seconds(10)); +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h new file mode 100644 index 0000000000..975b2ccd72 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h @@ -0,0 +1,39 @@ +#pragma once + +#include "types.h" +#include "responses.h" + +#include <kikimr/yndx/api/protos/persqueue.pb.h> + +#include <library/cpp/threading/future/future.h> + +#include <util/generic/vector.h> + +namespace NPersQueue { + +class IConsumer { +public: + virtual ~IConsumer() = default; + + // Start consumer. + // Consumer can be used after its start will be finished. + virtual NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept = 0; + + NThreading::TFuture<TConsumerCreateResponse> Start(TDuration timeout) noexcept { + return Start(TInstant::Now() + timeout); + } + + //read result according to settings or Lock/Release requests from server(if UseLockSession is true) + virtual NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept = 0; + + //commit processed reads + virtual void Commit(const TVector<ui64>& cookies) noexcept = 0; + + //will request status - response will be received in GetNextMessage with EMT_STATUS + virtual void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { Y_UNUSED(topic); Y_UNUSED(partition); Y_UNUSED(generation); }; + + //will be signalled in case of errors - you may use this future if you can't use GetNextMessage() //out of memory or any other reason + virtual NThreading::TFuture<TError> IsDead() noexcept = 0; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp new file mode 100644 index 0000000000..c29b1d8dd8 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.cpp @@ -0,0 +1,421 @@ +#include "channel.h" +#include "channel_p.h" +#include "internals.h" +#include "persqueue_p.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> + +#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> + +#include <util/random/random.h> +#include <util/string/builder.h> + +#include <grpc++/create_channel.h> + +const ui32 MAX_MESSAGE_SIZE = 150 * 1024 * 1024; +const TDuration KEEP_ALIVE_TIME = TDuration::Seconds(90); +const TDuration KEEP_ALIVE_TIMEOUT = TDuration::Seconds(1); +constexpr ui64 MAGIC_COOKIE_VALUE = 123456789; + +namespace NPersQueue { + +void FillMetaHeaders(grpc::ClientContext& context, const TString& database, ICredentialsProvider* credentials) { + context.AddMetadata(NYdb::YDB_AUTH_TICKET_HEADER, GetToken(credentials)); + context.AddMetadata(NYdb::YDB_DATABASE_HEADER, database); +} + +TServerSetting ApplyClusterEndpoint(const TServerSetting& server, const TString& endpoint) { + TServerSetting ret = server; + ret.UseLogbrokerCDS = EClusterDiscoveryUsageMode::DontUse; + TStringBuf endpointWithPort = endpoint; + TStringBuf host, port; + ui16 parsedPort = 0; + if (endpointWithPort.TryRSplit(':', host, port) && TryFromString(port, parsedPort)) { + ret.Address = TString(host); + ret.Port = parsedPort; + } else { + ret.Address = endpoint; + } + return ret; +} + +bool UseCDS(const TServerSetting& server) { + switch (server.UseLogbrokerCDS) { + case EClusterDiscoveryUsageMode::Auto: + return server.Address == "logbroker.yandex.net"sv || server.Address == "logbroker-prestable.yandex.net"sv; + case EClusterDiscoveryUsageMode::Use: + return true; + case EClusterDiscoveryUsageMode::DontUse: + return false; + } +} + +class TChannelImpl::TGetProxyHandler : public IHandler { +public: + TGetProxyHandler(TChannelImplPtr ptr) + : Ptr(ptr) + {} + + void Done() override { + Ptr->OnGetProxyDone(); + } + + void Destroy(const TError& reason) override { + Ptr->Destroy(reason); + } + + TString ToString() override { return "GetProxyHandler"; } + +protected: + TChannelImplPtr Ptr; +}; + +class TChannelOverCdsBaseImpl::TCdsResponseHandler : public IHandler { +public: + TCdsResponseHandler(TChannelOverCdsBaseImplPtr ptr) + : Ptr(ptr) + {} + + void Done() override { + Ptr->OnCdsRequestDone(); + } + + void Destroy(const TError& reason) override { + Ptr->Destroy(reason); + } + + TString ToString() override { return "CDS-ResponseHandler"; } + +protected: + TChannelOverCdsBaseImplPtr Ptr; +}; + +TChannel::TChannel( + const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy +) { + MakeImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy); +} + +TChannel::TChannel( + const TProducerSettings& settings, TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy +) { + if (UseCDS(settings.Server)) + Impl = new TProducerChannelOverCdsImpl(settings, pqLib, std::move(logger), preferLocalProxy); + else + MakeImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, preferLocalProxy); +} + +void TChannel::MakeImpl( + const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, + TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy +) { + Impl = MakeIntrusive<TChannelOverDiscoveryImpl>( + server, credentialsProvider, pqLib, std::move(logger), preferLocalProxy + ); +} + +NThreading::TFuture<TChannelInfo> TChannel::GetChannel() { + return Impl->GetChannel(); +} + +TChannel::~TChannel() { + Impl->TryCancel(); + Impl->Wait(); +} + +void TChannel::Start() { + Impl->Start(); +} + +TChannelImpl::TChannelImpl(const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, TPQLibPrivate* pqLib, TIntrusivePtr<ILogger> logger, bool preferLocalProxy) + : Promise(NThreading::NewPromise<TChannelInfo>()) + , Server(server) + , SelectedEndpoint(Server.GetFullAddressString()) + , CredentialsProvider(credentialsProvider) + , PreferLocalProxy(preferLocalProxy) + , Logger(std::move(logger)) + , PQLib(pqLib) + , CQ(pqLib->GetCompletionQueue()) + , ChooseProxyFinished(0) +{} + +TChannelOverDiscoveryImpl::TChannelOverDiscoveryImpl( + const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy +) + : TChannelImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy) +{ + CreationTimeout = PQLib->GetSettings().ChannelCreationTimeout; +} + +TChannelOverCdsBaseImpl::TChannelOverCdsBaseImpl(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy +) + : TChannelOverDiscoveryImpl(server, credentialsProvider, pqLib, logger, preferLocalProxy) + , CdsRequestFinished(0) +{} + +void TChannelOverDiscoveryImpl::Start() { + Channel = CreateGrpcChannel(SelectedEndpoint); + Stub = Ydb::Discovery::V1::DiscoveryService::NewStub(Channel); + IHandlerPtr handler = new TGetProxyHandler(this); + Context.set_deadline(TInstant::Now() + CreationTimeout); + Ydb::Discovery::ListEndpointsRequest request; + request.set_database(Server.Database); + FillMetaHeaders(Context, Server.Database, CredentialsProvider.get()); + DEBUG_LOG("Send list endpoints request: " << request, "", ""); + Rpc = Stub->AsyncListEndpoints(&Context, request, CQ.get()); + Rpc->Finish(&Response, &Status, new TQueueEvent(std::move(handler))); +} + +void TChannelOverCdsBaseImpl::Start() { + Channel = CreateGrpcChannel(Server.GetFullAddressString()); + Stub = Ydb::PersQueue::V1::ClusterDiscoveryService::NewStub(Channel); + IHandlerPtr handler = new TCdsResponseHandler(this); + CdsContext.set_deadline(TInstant::Now() + PQLib->GetSettings().ChannelCreationTimeout); + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request = GetCdsRequest(); + FillMetaHeaders(CdsContext, Server.Database, CredentialsProvider.get()); + DEBUG_LOG("Send cds request: " << request, "", ""); + CreationStartTime = TInstant::Now(); + Rpc = Stub->AsyncDiscoverClusters(&CdsContext, request, CQ.get()); + Rpc->Finish(&Response, &Status, new TQueueEvent(std::move(handler))); +} + +NThreading::TFuture<TChannelInfo> TChannelImpl::GetChannel() { + return Promise.GetFuture(); +} + +bool TChannelImpl::TryCancel() { + if (AtomicSwap(&ChooseProxyFinished, 1) == 0) { + Y_ASSERT(!Promise.HasValue()); + Promise.SetValue(TChannelInfo{nullptr, 0}); + Context.TryCancel(); + return true; + } + return false; +} + +TChannelImpl::~TChannelImpl() = default; + +std::shared_ptr<grpc::Channel> TChannelImpl::CreateGrpcChannel(const TString& address) { + grpc::ChannelArguments args; + args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, KEEP_ALIVE_TIME.MilliSeconds()); + args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, KEEP_ALIVE_TIMEOUT.MilliSeconds()); + args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); + args.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0); + args.SetInt(GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, KEEP_ALIVE_TIME.MilliSeconds()); + + args.SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, MAX_MESSAGE_SIZE); + args.SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, MAX_MESSAGE_SIZE); + + //TODO: fill there other settings from PQLib + DEBUG_LOG("Creating grpc channel to \"" << address << "\"", "", ""); + if (Server.UseSecureConnection) + return grpc::CreateCustomChannel(address, + grpc::SslCredentials(grpc::SslCredentialsOptions{Server.CaCert, "", ""}), + args); + else + return grpc::CreateCustomChannel(address, grpc::InsecureChannelCredentials(), args); +} + +TString TChannelImpl::GetProxyAddress(const TString& proxyName, ui32 port) { + TStringBuilder address; + if (proxyName.find(':') != TString::npos && proxyName.find('[') == TString::npos) { //fix for ipv6 adresses + address << "[" << proxyName << "]"; + } else { + address << proxyName; + } + if (port) + address << ":" << port; + else + address << ":" << Server.Port; + return std::move(address); +} + +void TChannelImpl::OnGetProxyDone() { + if (AtomicSwap(&ChooseProxyFinished, 1) == 0) { + const bool ok = Status.ok(); + Y_ASSERT(!Promise.HasValue()); + if (ok) { + ProcessGetProxyResponse(); + } else { + if (Status.error_code() == grpc::CANCELLED) { + INFO_LOG("Grpc request is canceled by user", "", ""); + } else { + const auto& msg = Status.error_message(); + TString res(msg.data(), msg.length()); + ERR_LOG("Grpc error " << static_cast<int>(Status.error_code()) << ": \"" << res << "\"", "", ""); + } + StartFailed(); + } + } + SetDone(); +} + +void TChannelImpl::StartFailed() { + AtomicSet(ChooseProxyFinished, 1); + Promise.SetValue(TChannelInfo{nullptr, 0}); +} + +void TChannelOverDiscoveryImpl::ProcessGetProxyResponse() { + Ydb::Discovery::ListEndpointsResult result; + auto& reply = Response.operation().result(); + reply.UnpackTo(&result); + + auto has_service = [](const Ydb::Discovery::EndpointInfo& endpoint) { + for (auto& service: endpoint.service()) { + if (service == "pq") + return true; + } + return false; + }; + TVector<std::pair<TString, ui32>> validEps; + for (const auto& ep : result.endpoints()) { + if (has_service(ep) && (!Server.UseSecureConnection || ep.ssl())) { + validEps.emplace_back(ep.address(), ep.port()); + } + } + if (validEps.empty()) { + ERR_LOG("No valid endpoints to connect!", "", ""); + StartFailed(); + } else { + auto& selected = validEps[RandomNumber<ui32>(validEps.size())]; + DEBUG_LOG("Selected endpoint {\"" << selected.first << "\", " << selected.second << "}", "", ""); + auto channel = CreateGrpcChannel(GetProxyAddress(selected.first, selected.second)); + Promise.SetValue(TChannelInfo{std::move(channel), MAGIC_COOKIE_VALUE}); + } +} + +void TChannelOverCdsBaseImpl::OnCdsRequestDone() { + DEBUG_LOG("ON CDS request done: " << Response, "", ""); + + if (AtomicSwap(&CdsRequestFinished, 1) == 0) { + const bool ok = Status.ok(); + if (ok) { + if (ProcessCdsResponse()) { + CreationTimeout = CreationTimeout - (TInstant::Now() - CreationStartTime); + TChannelOverDiscoveryImpl::Start(); + // Don't set AllDone event, because we are about to continue node discovery + // via TChannelOverDiscoveryImpl part of implementation. + return; + } else { + StartFailed(); + } + } else { + if (Status.error_code() == grpc::CANCELLED) { + INFO_LOG("Grpc request is canceled by user", "", ""); + } else { + const auto& msg = Status.error_message(); + TString res(msg.data(), msg.length()); + ERR_LOG("Grpc error " << static_cast<int>(Status.error_code()) << ": \"" << res << "\"", "", ""); + } + StartFailed(); + } + } + SetDone(); +} + +bool TChannelOverCdsBaseImpl::TryCancel() { + if (AtomicSwap(&CdsRequestFinished, 1) == 0) { + Y_ASSERT(!Promise.HasValue()); + Promise.SetValue(TChannelInfo{nullptr, 0}); + CdsContext.TryCancel(); + return true; + } else { + TChannelOverDiscoveryImpl::TryCancel(); + } + return false; +} + +TProducerChannelOverCdsImpl::TProducerChannelOverCdsImpl(const TProducerSettings& settings, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy) + : TChannelOverCdsBaseImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, preferLocalProxy) + , Topic(settings.Topic) + , SourceId(settings.SourceId) + , PreferredCluster(settings.PreferredCluster) +{ +} + +Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TProducerChannelOverCdsImpl::GetCdsRequest() const { + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; + auto* params = request.add_write_sessions(); + params->set_topic(Topic); + params->set_source_id(SourceId); + params->set_preferred_cluster_name(PreferredCluster); // Can be empty. + return request; +} + +bool TProducerChannelOverCdsImpl::ProcessCdsResponse() { + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult result; + auto& reply = Response.operation().result(); + reply.UnpackTo(&result); + DEBUG_LOG("Process CDS result: " << result, "", ""); + for (auto& session: result.write_sessions_clusters()) { + for (auto& cluster: session.clusters()) { + if (cluster.available()) { + const TServerSetting selectedServer = ApplyClusterEndpoint(Server, cluster.endpoint()); + SelectedEndpoint = GetProxyAddress(selectedServer.Address, selectedServer.Port); + return true; + } + } + } + ERR_LOG("Could not find valid cluster in CDS response", "", ""); + return false; +} + +TConsumerChannelOverCdsImpl::TConsumerChannelOverCdsImpl(const TConsumerSettings& settings, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger) + : TChannelOverCdsBaseImpl(settings.Server, settings.CredentialsProvider, pqLib, logger, true) + , Result(std::make_shared<TResult>()) + , Topics(settings.Topics) +{ +} + +Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TConsumerChannelOverCdsImpl::GetCdsRequest() const { + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest request; + for (const TString& topic : Topics) { + auto* params = request.add_read_sessions(); + params->set_topic(topic); + params->mutable_all_original(); // set all_original + } + return request; +} + +bool TConsumerChannelOverCdsImpl::ProcessCdsResponse() { + auto& reply = Response.operation().result(); + reply.UnpackTo(&Result->second); + Result->first = Status; + DEBUG_LOG("Got CDS result: " << Result->second, "", ""); + return false; // Signals future +} + +void TConsumerChannelOverCdsImpl::StartFailed() { + Result->first = Status; + TChannelOverCdsBaseImpl::StartFailed(); +} + +void TChannelImpl::SetDone() { + AllDoneEvent.Signal(); +} + +void TChannelImpl::Destroy(const TError& error) { + if (AtomicSwap(&ChooseProxyFinished, 1) == 0) { + const auto& msg = Status.error_message(); + TString res(msg.data(), msg.length()); + + ERR_LOG("Got proxy error response " << error << " grpc error " << res, "", ""); + //TODO: add here status from Status to response, log status + + Y_ASSERT(!Promise.HasValue()); + Promise.SetValue(TChannelInfo{nullptr, 0}); + } + + SetDone(); +} + +void TChannelImpl::Wait() { + AllDoneEvent.Wait(); +} + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h new file mode 100644 index 0000000000..630b097675 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h @@ -0,0 +1,50 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> + +#include <library/cpp/threading/future/future.h> + +#include <deque> + +namespace NPersQueue { + +struct TChannelInfo { + std::shared_ptr<grpc::Channel> Channel; + ui64 ProxyCookie; +}; + +class TChannel; +using TChannelPtr = TIntrusivePtr<TChannel>; + +class TPQLibPrivate; + +struct TChannelHolder { + TChannelPtr ChannelPtr; + NThreading::TFuture<TChannelInfo> ChannelInfo; +}; + +class TChannelImpl; +using TChannelImplPtr = TIntrusivePtr<TChannelImpl>; + +class TChannel: public TAtomicRefCount<TChannel> { +public: + friend class TPQLibPrivate; + + NThreading::TFuture<TChannelInfo> GetChannel(); + + ~TChannel(); + + void Start(); + +private: + TChannel(const TServerSetting& server, const std::shared_ptr<ICredentialsProvider>& credentialsProvider, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false); + TChannel(const TProducerSettings& settings, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false); + void MakeImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger = nullptr, bool preferLocalProxy = false); + TChannelImplPtr Impl; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h new file mode 100644 index 0000000000..10f18abb04 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel_p.h @@ -0,0 +1,156 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include "channel.h" +#include "scheduler.h" +#include "internals.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include <library/cpp/threading/future/future.h> +#include <ydb/public/api/protos/ydb_discovery.pb.h> +#include <ydb/public/api/grpc/ydb_discovery_v1.grpc.pb.h> +#include <ydb/public/api/grpc/draft/ydb_persqueue_v1.grpc.pb.h> + +#include <deque> + +namespace NPersQueue { + +class TPQLibPrivate; + +// Apply endpoint that was taken from CDS. +TServerSetting ApplyClusterEndpoint(const TServerSetting& server, const TString& endpoint); + +class TChannelImpl : public TAtomicRefCount<TChannelImpl> { +public: + NThreading::TFuture<TChannelInfo> GetChannel(); + + virtual ~TChannelImpl(); + + virtual bool TryCancel(); + + TChannelImpl(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy); + + virtual void Start() = 0; + + void Wait(); + +protected: + friend class TPQLibPrivate; + class TGetProxyHandler; + + void Destroy(const TError& error); + virtual void StartFailed(); + void SetDone(); + void OnGetProxyDone(); + virtual void ProcessGetProxyResponse() = 0; + std::shared_ptr<grpc::Channel> CreateGrpcChannel(const TString& address); + TString GetProxyAddress(const TString& proxyName, ui32 port = 0); + TString GetToken(); + + NThreading::TPromise<TChannelInfo> Promise; + TServerSetting Server; + TString SelectedEndpoint; + std::shared_ptr<ICredentialsProvider> CredentialsProvider; + bool PreferLocalProxy; + + TIntrusivePtr<ILogger> Logger; + TPQLibPrivate* PQLib; + +protected: + std::shared_ptr<grpc::CompletionQueue> CQ; + std::shared_ptr<grpc::Channel> Channel; + grpc::ClientContext Context; + grpc::Status Status; + + TAutoEvent AllDoneEvent; + + TAtomic ChooseProxyFinished; +}; + +class TChannelOverDiscoveryImpl : public TChannelImpl { +public: + TChannelOverDiscoveryImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy); + void Start() override; + +protected: + void ProcessGetProxyResponse() override; + TDuration CreationTimeout; + +private: + std::unique_ptr<Ydb::Discovery::V1::DiscoveryService::Stub> Stub; + std::unique_ptr<grpc::ClientAsyncResponseReader<Ydb::Discovery::ListEndpointsResponse>> Rpc; + Ydb::Discovery::ListEndpointsResponse Response; +}; + + +class TChannelOverCdsBaseImpl : public TChannelOverDiscoveryImpl { +public: + TChannelOverCdsBaseImpl(const TServerSetting& server, const TCredProviderPtr&, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy); + void Start() override; + void OnCdsRequestDone(); + + bool TryCancel() override; + +private: + virtual Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const = 0; + virtual bool ProcessCdsResponse() = 0; + +protected: + class TCdsResponseHandler; + +private: + std::unique_ptr<Ydb::PersQueue::V1::ClusterDiscoveryService::Stub> Stub; + std::unique_ptr<grpc::ClientAsyncResponseReader<Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse>> Rpc; + +protected: + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResponse Response; + +private: + grpc::ClientContext CdsContext; + + TInstant CreationStartTime = TInstant::Zero(); + TAtomic CdsRequestFinished; +}; +using TChannelOverCdsBaseImplPtr = TIntrusivePtr<TChannelOverCdsBaseImpl>; + +class TProducerChannelOverCdsImpl : public TChannelOverCdsBaseImpl { +public: + TProducerChannelOverCdsImpl(const TProducerSettings& settings, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy); + +private: + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const override; + bool ProcessCdsResponse() override; + +private: + TString Topic; + TString SourceId; + TString PreferredCluster; +}; + +class TConsumerChannelOverCdsImpl : public TChannelOverCdsBaseImpl { +public: + using TResult = std::pair<grpc::Status, Ydb::PersQueue::ClusterDiscovery::DiscoverClustersResult>; + using TResultPtr = std::shared_ptr<TResult>; + +public: + TConsumerChannelOverCdsImpl(const TConsumerSettings& settings, TPQLibPrivate* pqLib, + TIntrusivePtr<ILogger> logger); + + TResultPtr GetResultPtr() const { + return Result; + } + +private: + Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest GetCdsRequest() const override; + bool ProcessCdsResponse() override; + void StartFailed() override; + +private: + TResultPtr Result; + TVector<TString> Topics; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp new file mode 100644 index 0000000000..14c2871a54 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.cpp @@ -0,0 +1,256 @@ + +#include "compat_producer.h" +#include "persqueue_p.h" + +namespace NPersQueue { + + using namespace NYdb::NPersQueue; + + + static TWriteSessionSettings ConvertToWriteSessionSettings(const TProducerSettings& ps) { + TWriteSessionSettings settings; + settings.Path(ps.Topic) + .MessageGroupId(ps.SourceId) + .RetryPolicy(ps.ReconnectOnFailure ? NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy(ps.ReconnectionDelay, ps.ReconnectionDelay, ps.MaxReconnectionDelay, + ps.MaxAttempts, TDuration::Max(), 2.0, [](NYdb::EStatus){ return ERetryErrorClass::ShortRetry; }) + : NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()) + .Codec(NYdb::NPersQueue::ECodec(ps.Codec + 1)) + .CompressionLevel(ps.Quality) + .ValidateSeqNo(false); + for (auto& attr : ps.ExtraAttrs) { + settings.AppendSessionMeta(attr.first, attr.second); + } + + if (ps.PreferredCluster) { + settings.PreferredCluster(ps.PreferredCluster); + } + if (ps.DisableCDS || ps.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::DontUse) { + settings.ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Off); + } else if (ps.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::Use) { + settings.ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::On); + } + + return settings; + } + + TYdbSdkCompatibilityProducer::~TYdbSdkCompatibilityProducer() { + WriteSession->Close(TDuration::Seconds(0)); + + NotifyClient(NErrorCode::OK, "producer object destroyed by client"); + } + + TYdbSdkCompatibilityProducer::TYdbSdkCompatibilityProducer(const TProducerSettings& settings, TPersQueueClient& persQueueClient, + std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) + : IProducerImpl(destroyEventRef, pqLib) + { + IsDeadPromise = ::NThreading::NewPromise<TError>(); + TWriteSessionSettings sessionSettings = ConvertToWriteSessionSettings(settings); + Closed = false; + WriteSession = persQueueClient.CreateWriteSession(sessionSettings); + } + + void TYdbSdkCompatibilityProducer::SubscribeToNextEvent() { + NextEvent = WriteSession->WaitEvent(); + std::weak_ptr<TYdbSdkCompatibilityProducer> self = shared_from_this(); + NextEvent.Subscribe([self](const auto& future) { + auto selfShared = self.lock(); + Y_UNUSED(future); + if (selfShared) { + selfShared->DoProcessNextEvent(); + } + }); + } + + void TYdbSdkCompatibilityProducer::Init() noexcept { + } + + NThreading::TFuture<TProducerCreateResponse> TYdbSdkCompatibilityProducer::Start(TInstant deadline) noexcept { + Y_UNUSED(deadline); // TODO: start timeout? + SubscribeToNextEvent(); + return WriteSession->GetInitSeqNo().Apply([](const auto& future) { + TWriteResponse response; + if (future.HasException()) { + response.MutableError()->SetDescription("session closed"); + response.MutableError()->SetCode(NErrorCode::ERROR); + } else { + response.MutableInit()->SetMaxSeqNo(future.GetValue()); + } + TProducerCreateResponse res(std::move(response)); + + return NThreading::MakeFuture<TProducerCreateResponse>(res); + }); + } + + void TYdbSdkCompatibilityProducer::WriteImpl(NThreading::TPromise<TProducerCommitResponse>& promise, TMaybe<TProducerSeqNo> seqNo, ::NPersQueue::TData data) noexcept { + TContinuationToken *contToken = nullptr; + bool write = false; + with_lock(Lock) { + if (!Closed) { + contToken = ContToken.Get(); + if(contToken == nullptr || !ToWrite.empty()) { + ToWrite.emplace(promise, seqNo, data); + return; + } else { // contToken here and ToWrite is empty - can write right now + ToAck.emplace(promise, seqNo, data); + write = true; + } + } + } + if (write) { + if (data.IsEncoded()) { + WriteSession->WriteEncoded(std::move(*contToken), data.GetEncodedData(), NYdb::NPersQueue::ECodec(data.GetCodecType() + 1), data.GetOriginalSize(), seqNo); // can trigger NextEvent.Subscribe() in the same thread + } else { + WriteSession->Write(std::move(*contToken), data.GetSourceData(), seqNo); // can trigger NextEvent.Subscribe() in the same thread + } + } else { + TWriteResponse wr; + wr.MutableError()->SetCode(NErrorCode::ERROR); + wr.MutableError()->SetDescription("session closed"); + TProducerCommitResponse resp(seqNo ? *seqNo : 0, data, std::move(wr)); + promise.SetValue(resp); + } + } + + void TYdbSdkCompatibilityProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, ::NPersQueue::TData data) noexcept { + return WriteImpl(promise, {}, std::move(data)); // similar to TCompressingProducer + } + + void TYdbSdkCompatibilityProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, ::NPersQueue::TData data) noexcept { + return WriteImpl(promise, seqNo, std::move(data)); // similar to TCompressingProducer + } + + void TYdbSdkCompatibilityProducer::Cancel() { + WriteSession->Close(TDuration::Seconds(0)); + } + + NErrorCode::EErrorCode GetErrorCode(const NYdb::EStatus status) { + switch(status) { + case NYdb::EStatus::SUCCESS: + return NErrorCode::OK; + case NYdb::EStatus::UNAVAILABLE: + return NErrorCode::INITIALIZING; + case NYdb::EStatus::OVERLOADED: + return NErrorCode::OVERLOAD; + case NYdb::EStatus::BAD_REQUEST: + return NErrorCode::BAD_REQUEST; + case NYdb::EStatus::NOT_FOUND: + case NYdb::EStatus::SCHEME_ERROR: + return NErrorCode::UNKNOWN_TOPIC; + case NYdb::EStatus::UNSUPPORTED: + return NErrorCode::BAD_REQUEST; + case NYdb::EStatus::UNAUTHORIZED: + return NErrorCode::ACCESS_DENIED; + default: + return NErrorCode::ERROR; + } + return NErrorCode::ERROR; + } + + struct TToNotify { + NThreading::TPromise<TProducerCommitResponse> Promise; + TProducerCommitResponse Resp; + + TToNotify(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerCommitResponse&& resp) + : Promise(promise) + , Resp(resp) + {} + }; + + + void TYdbSdkCompatibilityProducer::NotifyClient(NErrorCode::EErrorCode code, const TString& reason) { + + TError err; + err.SetCode(code); + err.SetDescription(reason); + std::queue<TToNotify> toNotify; + with_lock(Lock) { + if (Closed) return; + Closed = true; + while(!ToAck.empty()) { + TMsgData& item = ToAck.front(); + TWriteResponse wr; + wr.MutableError()->CopyFrom(err); + TProducerCommitResponse resp(item.SeqNo ? *item.SeqNo : 0, item.Data, std::move(wr)); + toNotify.emplace(item.Promise, std::move(resp)); + ToAck.pop(); + } + while(!ToWrite.empty()) { + TMsgData& item = ToWrite.front(); + TWriteResponse wr; + wr.MutableError()->CopyFrom(err); + TProducerCommitResponse resp(item.SeqNo ? *item.SeqNo : 0, item.Data, std::move(wr)); + toNotify.emplace(item.Promise, std::move(resp)); + ToWrite.pop(); + } + } + while(!toNotify.empty()) { + toNotify.front().Promise.SetValue(toNotify.front().Resp); + toNotify.pop(); + } + IsDeadPromise.SetValue(err); + DestroyPQLibRef(); + } + + void TYdbSdkCompatibilityProducer::DoProcessNextEvent() { + + std::queue<TToNotify> toNotify; + + TVector<TWriteSessionEvent::TEvent> events = WriteSession->GetEvents(false); + //Y_VERIFY(!events.empty()); + for (auto& event : events) { + if(std::holds_alternative<TSessionClosedEvent>(event)) { + NotifyClient(GetErrorCode(std::get<TSessionClosedEvent>(event).GetStatus()), std::get<TSessionClosedEvent>(event).DebugString() ); + return; + } else if(std::holds_alternative<TWriteSessionEvent::TAcksEvent>(event)) { + + // get seqNos, signal their promises + TWriteSessionEvent::TAcksEvent& acksEvent = std::get<TWriteSessionEvent::TAcksEvent>(event); + std::queue<TToNotify> toNotify; + for(TWriteSessionEvent::TWriteAck& ackEvent : acksEvent.Acks) { + TWriteResponse writeResp; + auto ackMsg = writeResp.MutableAck(); + ackMsg->SetSeqNo(ackEvent.SeqNo); + ackMsg->SetAlreadyWritten(ackEvent.State == TWriteSessionEvent::TWriteAck::EEventState::EES_ALREADY_WRITTEN); + ackMsg->SetOffset(ackEvent.Details ? ackEvent.Details->Offset : 0); + with_lock(Lock) { + Y_ASSERT(!ToAck.empty()); + TProducerCommitResponse resp(ackEvent.SeqNo, ToAck.front().Data, std::move(writeResp)); + toNotify.emplace(ToAck.front().Promise, std::move(resp)); + ToAck.pop(); + } + } + while(!toNotify.empty()) { + toNotify.front().Promise.SetValue(toNotify.front().Resp); + toNotify.pop(); + } + } else if(std::holds_alternative<TWriteSessionEvent::TReadyToAcceptEvent>(event)) { + TWriteSessionEvent::TReadyToAcceptEvent& readyEvent = std::get<TWriteSessionEvent::TReadyToAcceptEvent>(event); + TContinuationToken contToken = std::move(readyEvent.ContinuationToken); // IF there is only one only movable contToken THEN this is thread safe + TString strData; + TMaybe<TProducerSeqNo> seqNo; + bool write = false; + with_lock(Lock) { + if(ToWrite.empty()) { + ContToken = std::move(contToken); + } else { + strData = ToWrite.front().Data.GetSourceData(); + seqNo = ToWrite.front().SeqNo; + ToAck.push(ToWrite.front()); + ToWrite.pop(); + write = true; + } + } + if (write) { + WriteSession->Write(std::move(contToken), strData, seqNo); + } + } + } + + SubscribeToNextEvent(); + } + + NThreading::TFuture<TError> TYdbSdkCompatibilityProducer::IsDead() noexcept { + return IsDeadPromise.GetFuture(); + } + +} // namespace NYdb::NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h new file mode 100644 index 0000000000..e5287d4694 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compat_producer.h @@ -0,0 +1,66 @@ +#pragma once + +#include "iproducer_p.h" + +#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h> + +#include <memory> +#include <queue> + +namespace NPersQueue { + + +/** + * Compatibility producer: + * wraps old "interface" around the new one + */ + +class TYdbSdkCompatibilityProducer: public IProducerImpl, public std::enable_shared_from_this<TYdbSdkCompatibilityProducer> { + + struct TMsgData { + NThreading::TPromise<TProducerCommitResponse> Promise; + TMaybe<TProducerSeqNo> SeqNo; + TData Data; + + TMsgData() {}; + TMsgData(NThreading::TPromise<TProducerCommitResponse> promise, TMaybe<TProducerSeqNo> seqNo, TData data) + : Promise(promise) + , SeqNo(seqNo) + , Data(data) {} + }; + + std::shared_ptr<NYdb::NPersQueue::IWriteSession> WriteSession; + NThreading::TPromise<TError> IsDeadPromise; + NThreading::TFuture<void> NextEvent; + + TMaybe<NYdb::NPersQueue::TContinuationToken> ContToken; + std::queue<TMsgData> ToWrite; + std::queue<TMsgData> ToAck; + TSpinLock Lock; + bool Closed; + + void DoProcessNextEvent(); + void SubscribeToNextEvent(); + + void WriteImpl(NThreading::TPromise<TProducerCommitResponse>& promise, TMaybe<TProducerSeqNo> seqNo, TData data) noexcept; + + void NotifyClient(NErrorCode::EErrorCode code, const TString& reason); + +public: + using IProducerImpl::Write; + NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept override; + NThreading::TFuture<TError> IsDead() noexcept override; + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override; + void Cancel() override; + void Init() noexcept override; + + ~TYdbSdkCompatibilityProducer(); + + TYdbSdkCompatibilityProducer(const TProducerSettings& settings, NYdb::NPersQueue::TPersQueueClient& persQueueClient, + std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); + +}; + + +} // namespace NYdb::NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp new file mode 100644 index 0000000000..ecb6f70944 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.cpp @@ -0,0 +1,155 @@ +#include "compressing_producer.h" +#include "persqueue_p.h" + +namespace NPersQueue { + +TCompressingProducer::TCompressingProducer(std::shared_ptr<IProducerImpl> subproducer, ECodec defaultCodec, int quality, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) + : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) + , Logger(std::move(logger)) + , Subproducer(std::move(subproducer)) + , DefaultCodec(defaultCodec) + , Quality(quality) +{ + if (defaultCodec == ECodec::DEFAULT) { + ythrow yexception() << "Producer codec can't be ECodec::DEFAULT"; + } +} + +TCompressingProducer::~TCompressingProducer() { + DestroyQueue("Destructor called"); +} + +NThreading::TFuture<TProducerCreateResponse> TCompressingProducer::Start(TInstant deadline) noexcept { + return Subproducer->Start(deadline); +} + +NThreading::TFuture<TData> TCompressingProducer::Enqueue(TData data) { + NThreading::TPromise<TData> promise = NThreading::NewPromise<TData>(); + std::weak_ptr<TCompressingProducer> self = shared_from_this(); + const void* queueTag = this; + const ECodec defaultCodec = DefaultCodec; + const int quality = Quality; + auto compress = [promise, self, data, queueTag, defaultCodec, quality, pqLib = PQLib.Get()]() mutable { + Compress(promise, std::move(data), defaultCodec, quality, queueTag, self, pqLib); + }; + Y_VERIFY(PQLib->GetCompressionPool().AddFunc(compress)); + return promise.GetFuture(); +} + +void TCompressingProducer::Compress(NThreading::TPromise<TData>& promise, TData&& data, ECodec defaultCodec, int quality, const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib) { + if (!self.lock()) { + return; + } + promise.SetValue(TData::Encode(std::move(data), defaultCodec, quality)); + SignalProcessQueue(queueTag, self, pqLib); +} + +void TCompressingProducer::SignalProcessQueue(const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib) { + auto processQueue = [self] { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->ProcessQueue(); + } + }; + + //can't use this->Logger here - weak_ptr::lock -> ~TCompressingProducer in compression thread -> + //race with cancel() in main thread pool + Y_UNUSED(pqLib->GetQueuePool().GetQueue(queueTag).AddFunc(processQueue)); +} + +void TCompressingProducer::ProcessQueue() { + if (DestroyedFlag) { + return; + } + + size_t removeCount = 0; + bool prevSignalled = true; // flag to guarantee correct order of futures signalling. + for (TWriteRequestInfo& request : Queue) { + if (!request.EncodedData.HasValue()) { + // stage 1 + break; + } + if (!request.Future.Initialized()) { + // stage 2 + if (request.SeqNo) { + request.Future = Subproducer->Write(request.SeqNo, request.EncodedData.GetValue()); + } else { + request.Future = Subproducer->Write(request.EncodedData.GetValue()); + } + std::weak_ptr<TCompressingProducer> self = shared_from_this(); + const void* queueTag = this; + auto signalProcess = [self, queueTag, pqLib = PQLib.Get()](const auto&) { + SignalProcessQueue(queueTag, self, pqLib); + }; + request.Future.Subscribe(signalProcess); + } else { + // stage 3 + if (prevSignalled && request.Future.HasValue()) { + request.Promise.SetValue(request.Future.GetValue()); + ++removeCount; + } else { + prevSignalled = false; + } + } + } + while (removeCount--) { + Queue.pop_front(); + } +} + +void TCompressingProducer::DestroyQueue(const TString& reason) { + if (DestroyedFlag) { + return; + } + + DestroyedFlag = true; + for (TWriteRequestInfo& request : Queue) { + if (request.Future.Initialized() && request.Future.HasValue()) { + request.Promise.SetValue(request.Future.GetValue()); + } else { + TWriteResponse resp; + TError& err = *resp.MutableError(); + err.SetCode(NErrorCode::ERROR); + err.SetDescription(reason); + request.Promise.SetValue(TProducerCommitResponse(request.SeqNo, std::move(request.Data), std::move(resp))); + } + } + Queue.clear(); + + DestroyPQLibRef(); +} + +void TCompressingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept { + if (DestroyedFlag) { + return; + } + + Queue.emplace_back(seqNo, data, promise); + if (data.IsEncoded()) { + Queue.back().EncodedData = NThreading::MakeFuture<TData>(data); + ProcessQueue(); + } else if (DefaultCodec == ECodec::RAW) { + Queue.back().EncodedData = NThreading::MakeFuture<TData>(TData::MakeRawIfNotEncoded(data)); + ProcessQueue(); + } else { + Queue.back().EncodedData = Enqueue(std::move(data)); + } +} + +void TCompressingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept { + TCompressingProducer::Write(promise, 0, std::move(data)); +} + +NThreading::TFuture<TError> TCompressingProducer::IsDead() noexcept { + return Subproducer->IsDead(); +} + +NThreading::TFuture<void> TCompressingProducer::Destroyed() noexcept { + return WaitExceptionOrAll(DestroyedPromise.GetFuture(), Subproducer->Destroyed()); +} + +void TCompressingProducer::Cancel() { + DestroyQueue(GetCancelReason()); +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h new file mode 100644 index 0000000000..6ed9e89bed --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer.h @@ -0,0 +1,64 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include "internals.h" +#include "iproducer_p.h" + +#include <deque> +#include <memory> + +namespace NPersQueue { + +class TPQLibPrivate; + +class TCompressingProducer: public IProducerImpl, public std::enable_shared_from_this<TCompressingProducer> { +public: + NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; + using IProducerImpl::Write; + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override; + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; + NThreading::TFuture<TError> IsDead() noexcept override; + NThreading::TFuture<void> Destroyed() noexcept override; + + ~TCompressingProducer(); + + TCompressingProducer(std::shared_ptr<IProducerImpl> subproducer, ECodec defaultCodec, int quality, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); + + void Cancel() override; + +private: + struct TWriteRequestInfo { + TWriteRequestInfo(TProducerSeqNo seqNo, TData data, const NThreading::TPromise<TProducerCommitResponse>& promise) + : SeqNo(seqNo) + , Data(std::move(data)) + , Promise(promise) + { + } + + TProducerSeqNo SeqNo; + TData Data; + NThreading::TFuture<TData> EncodedData; + NThreading::TPromise<TProducerCommitResponse> Promise; + NThreading::TFuture<TProducerCommitResponse> Future; + }; + + void ProcessQueue(); + NThreading::TFuture<TData> Enqueue(TData data); + // these functions should be static and not lock self in compression thread pool for proper futures ordering + static void Compress(NThreading::TPromise<TData>& promise, TData&& data, ECodec defaultCodec, int quality, + const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib); + static void SignalProcessQueue(const void* queueTag, std::weak_ptr<TCompressingProducer> self, TPQLibPrivate* pqLib); + void DestroyQueue(const TString& reason); + +protected: + TIntrusivePtr<ILogger> Logger; + std::shared_ptr<IProducerImpl> Subproducer; + ECodec DefaultCodec; + int Quality; + std::deque<TWriteRequestInfo> Queue; + bool DestroyedFlag = false; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp new file mode 100644 index 0000000000..4d9b30f69a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/compressing_producer_ut.cpp @@ -0,0 +1,243 @@ +#include "compressing_producer.h" +#include "local_caller.h" +#include "persqueue_p.h" + +#include <library/cpp/testing/gmock_in_unittest/gmock.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/thread.h> + +using namespace testing; + +namespace NPersQueue { + +class TMockProducer: public IProducerImpl { +public: + TMockProducer() + : IProducerImpl(nullptr, nullptr) + { + } + + NThreading::TFuture<TProducerCreateResponse> Start(TInstant) noexcept override { + return NThreading::MakeFuture<TProducerCreateResponse>(MockStart()); + } + + NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept override { + return NThreading::MakeFuture<TProducerCommitResponse>(MockWriteWithSeqNo(seqNo, data)); + } + + NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept override { + return NThreading::MakeFuture<TProducerCommitResponse>(MockWrite(data)); + } + + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override { + promise.SetValue(MockWriteWithSeqNo(seqNo, data)); + } + + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override { + promise.SetValue(MockWrite(data)); + } + + NThreading::TFuture<TError> IsDead() noexcept override { + return MockIsDead(); + } + + void Cancel() override { + } + + MOCK_METHOD(TProducerCreateResponse, MockStart, (), ()); + MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ()); + MOCK_METHOD(TProducerCommitResponse, MockWriteWithSeqNo, (TProducerSeqNo, TData), ()); + MOCK_METHOD(TProducerCommitResponse, MockWrite, (TData), ()); +}; + +template <class TMock = TMockProducer> +struct TCompressingProducerBootstrap { + ~TCompressingProducerBootstrap() { + Lib->CancelObjectsAndWait(); + } + + void Create(ECodec codec = ECodec::GZIP) { + Lib = new TPQLibPrivate(Settings); + MockProducer = std::make_shared<TMock>(); + CompressingProducer = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(MockProducer, codec, 3, Lib->GetSelfRefsAreDeadPtr(), Lib, Logger); + Lib->AddToDestroySet(CompressingProducer); + } + + void MakeOKStartResponse() { + TWriteResponse wresp; + wresp.MutableInit()->SetTopic("trololo"); + TProducerCreateResponse resp(std::move(wresp)); + EXPECT_CALL(*MockProducer, MockStart()) + .WillOnce(Return(resp)); + } + + void ExpectIsDeadCall() { + EXPECT_CALL(*MockProducer, MockIsDead()) + .WillOnce(Return(DeadPromise.GetFuture())); + } + + void Start() { + MakeOKStartResponse(); + UNIT_ASSERT_STRINGS_EQUAL(CompressingProducer->Start().GetValueSync().Response.GetInit().GetTopic(), "trololo"); + } + + TPQLibSettings Settings; + TIntrusivePtr<TCerrLogger> Logger = new TCerrLogger(TLOG_DEBUG); + TIntrusivePtr<TPQLibPrivate> Lib; + std::shared_ptr<TMock> MockProducer; + std::shared_ptr<IProducerImpl> CompressingProducer; + NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); +}; + +Y_UNIT_TEST_SUITE(TCompressingProducerTest) { + Y_UNIT_TEST(PassesStartAndDead) { + TCompressingProducerBootstrap<> bootstrap; + bootstrap.Create(); + bootstrap.ExpectIsDeadCall(); + + TWriteResponse wresp; + wresp.MutableError()->SetDescription("trololo"); + TProducerCreateResponse resp(std::move(wresp)); + EXPECT_CALL(*bootstrap.MockProducer, MockStart()) + .WillOnce(Return(resp)); + + auto startFuture = bootstrap.CompressingProducer->Start(); + UNIT_ASSERT_STRINGS_EQUAL(startFuture.GetValueSync().Response.GetError().GetDescription(), "trololo"); + + auto isDeadFuture = bootstrap.CompressingProducer->IsDead(); + UNIT_ASSERT(isDeadFuture.Initialized()); + UNIT_ASSERT(!isDeadFuture.HasValue()); + + TError err; + err.SetDescription("42"); + bootstrap.DeadPromise.SetValue(err); + + UNIT_ASSERT_STRINGS_EQUAL(isDeadFuture.GetValueSync().GetDescription(), "42"); + } + + Y_UNIT_TEST(CallsWritesInProperOrder) { + TCompressingProducerBootstrap<> bootstrap; + bootstrap.Create(); + bootstrap.Start(); + + TData data1 = TData("data1"); + TData expectedData1 = TData::Encode(data1, ECodec::GZIP, 3); + + TData data100 = TData::Raw("data100"); + TData expectedData100 = data100; + + TData data101 = TData("data101", ECodec::LZOP); + TData expectedData101 = TData::Encode(data101, ECodec::GZIP, 3); + + TWriteResponse wresp; + wresp.MutableAck(); + + InSequence seq; + EXPECT_CALL(*bootstrap.MockProducer, MockWriteWithSeqNo(1, expectedData1)) + .WillOnce(Return(TProducerCommitResponse(1, expectedData1, TWriteResponse(wresp)))); + + EXPECT_CALL(*bootstrap.MockProducer, MockWriteWithSeqNo(100, expectedData100)) + .WillOnce(Return(TProducerCommitResponse(100, expectedData100, TWriteResponse(wresp)))); + + EXPECT_CALL(*bootstrap.MockProducer, MockWrite(expectedData101)) + .WillOnce(Return(TProducerCommitResponse(101, expectedData101, TWriteResponse(wresp)))); + + auto write1 = bootstrap.CompressingProducer->Write(1, data1); + auto write100 = bootstrap.CompressingProducer->Write(100, data100); + auto write101 = bootstrap.CompressingProducer->Write(data101); + + write1.GetValueSync(); + write100.GetValueSync(); + write101.GetValueSync(); + } + + class TMockProducerWithReorder: public TMockProducer { + public: + using TMockProducer::TMockProducer; + using TMockProducer::Write; + + NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept override { + return PromiseWriteWithSeqNo(seqNo, data); + } + + MOCK_METHOD(NThreading::TFuture<TProducerCommitResponse>, PromiseWriteWithSeqNo, (TProducerSeqNo, TData), ()); + }; + + void SignalsAllFuturesInProperOrderImpl(bool death) { + size_t lastWritten = 0; + + TCompressingProducerBootstrap<TMockProducerWithReorder> bootstrap; + bootstrap.Create(); + bootstrap.Start(); + + const size_t count = 200; + NThreading::TFuture<TProducerCommitResponse> responses[count]; + TData datas[count]; + + TWriteResponse wresp; + wresp.MutableAck(); + EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(_, _)) + .Times(AtLeast(0)) + .WillRepeatedly(Return(NThreading::MakeFuture<TProducerCommitResponse>(TProducerCommitResponse(1, TData("data"), TWriteResponse(wresp))))); + + NThreading::TPromise<TProducerCommitResponse> promise = NThreading::NewPromise<TProducerCommitResponse>(); + + if (death) { + EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(50, _)) + .Times(AtLeast(0)) + .WillRepeatedly(Return(promise.GetFuture())); + } else { + EXPECT_CALL(*bootstrap.MockProducer, PromiseWriteWithSeqNo(50, _)) + .Times(1) + .WillRepeatedly(Return(promise.GetFuture())); + } + + TString bigString(10000, 'A'); + for (size_t i = 0; i < count; ++i) { + TData data(TStringBuilder() << bigString << i, ECodec::LZOP); + datas[i] = data; + } + + auto testThreadId = TThread::CurrentThreadId(), prevThreadId = 0ul; + for (size_t i = 0; i < count; ++i) { + size_t seqNo = i + 1; + responses[i] = bootstrap.CompressingProducer->Write(i + 1, datas[i]); + auto onSignal = [&lastWritten, &prevThreadId, seqNo, testThreadId](const auto&) { + // proper thread + auto curId = TThread::CurrentThreadId(); + if (prevThreadId != 0) { + UNIT_ASSERT_C(curId == prevThreadId || curId == testThreadId, // could be executed in unittest thread if future ia already signalled or in object thread + "prevThreadId: " << Hex(prevThreadId) << ", curId: " << Hex(curId) << ", testThreadId: " << Hex(testThreadId)); + } else if (curId != testThreadId) { + prevThreadId = curId; + } + + // proper order + UNIT_ASSERT_VALUES_EQUAL(seqNo, lastWritten + 1); + lastWritten = seqNo; + }; + responses[i].Subscribe(onSignal); + } + + if (death) { + bootstrap.CompressingProducer = nullptr; + } + + responses[45].GetValueSync(); + promise.SetValue(TProducerCommitResponse(49, TData("data"), TWriteResponse(wresp))); + responses[count - 1].GetValueSync(); + for (size_t i = count - 1; i > 0; --i) { + UNIT_ASSERT(responses[i - 1].HasValue()); + } + } + + Y_UNIT_TEST(SignalsAllFuturesInProperOrderDuringDeath) { + SignalsAllFuturesInProperOrderImpl(true); + } + + Y_UNIT_TEST(SignalsAllFuturesInProperOrder) { + SignalsAllFuturesInProperOrderImpl(false); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp new file mode 100644 index 0000000000..da5536ab6f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.cpp @@ -0,0 +1,612 @@ +#include "consumer.h" +#include "channel.h" +#include "persqueue_p.h" + +#include <kikimr/yndx/persqueue/read_batch_converter/read_batch_converter.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +#include <util/generic/strbuf.h> +#include <util/string/cast.h> +#include <util/string/printf.h> +#include <util/string/vector.h> +#include <util/string/builder.h> + +#include <grpc++/create_channel.h> + +namespace NPersQueue { + +class TConsumerDestroyHandler : public IHandler { +public: + TConsumerDestroyHandler(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : Ptr(std::move(ptr)) + , RpcStuff(std::move(rpcStuff)) + , QueueTag(queueTag) + , PQLib(pqLib) + {} + + void Destroy(const TError& reason) override { + auto consumer = Ptr; + auto handler = [consumer, reason] { + auto consumerShared = consumer.lock(); + if (consumerShared) { + consumerShared->Destroy(reason); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + + TString ToString() override { return "DestroyHandler"; } + +protected: + std::weak_ptr<TConsumer> Ptr; + TIntrusivePtr<TConsumer::TRpcStuff> RpcStuff; // must simply live + const void* QueueTag; + TPQLibPrivate* PQLib; +}; + +class TConsumerStreamCreated : public TConsumerDestroyHandler { +public: + TConsumerStreamCreated(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) + {} + + void Done() override { + auto consumer = Ptr; + auto handler = [consumer] { + auto consumerShared = consumer.lock(); + if (consumerShared) { + consumerShared->OnStreamCreated(); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + + TString ToString() override { return "StreamCreated"; } +}; + +class TConsumerWriteDone : public TConsumerDestroyHandler { +public: + TConsumerWriteDone(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) + {} + + void Done() override { + auto consumer = Ptr; + auto handler = [consumer] { + auto consumerShared = consumer.lock(); + if (consumerShared) { + consumerShared->OnWriteDone(); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + + TString ToString() override { return "WriteDone"; } +}; + +class TConsumerReadDone : public TConsumerDestroyHandler { +public: + TConsumerReadDone(std::weak_ptr<TConsumer> ptr, TIntrusivePtr<TConsumer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : TConsumerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) + {} + + void Done() override { + auto consumer = Ptr; + auto handler = [consumer] { + auto consumerShared = consumer.lock(); + if (consumerShared) { + consumerShared->OnReadDone(); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + + TString ToString() override { return "ReadDone"; } +}; + +void TConsumer::Destroy() noexcept { + IsDestroying = true; + RpcStuff->Context.TryCancel(); + ChannelHolder.ChannelPtr = nullptr; //if waiting for channel creation + Destroy("Destructor called"); +} + +TConsumer::TConsumer(const TConsumerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, + NThreading::TPromise<TConsumerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept + : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) + , RpcStuff(new TRpcStuff()) + , Settings(settings) + , StartPromise(promise) + , IsDeadPromise(NThreading::NewPromise<TError>()) + , UncommittedCount(0) + , UncommittedSize(0) + , MemoryUsage(0) + , ReadsOrdered(0) + , EstimateReadSize(settings.MaxSize) + , WriteInflight(true) + , ProxyCookie(0) + , Logger(std::move(logger)) + , IsDestroyed(false) + , IsDestroying(false) +{ + RpcStuff->CQ = std::move(cq); +} + +void TConsumer::Init() +{ + std::weak_ptr<TConsumer> self(shared_from_this()); + StreamCreatedHandler.Reset(new TConsumerStreamCreated(self, RpcStuff, this, PQLib.Get())); + ReadDoneHandler.Reset(new TConsumerReadDone(self, RpcStuff, this, PQLib.Get())); + WriteDoneHandler.Reset(new TConsumerWriteDone(self, RpcStuff, this, PQLib.Get())); +} + +void TConsumer::DoStart(TInstant deadline) +{ + if (IsDestroyed) + return; + + if (deadline != TInstant::Max()) { + std::weak_ptr<TConsumer> self = shared_from_this(); + auto onStartTimeout = [self] { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnStartTimeout(); + } + }; + StartDeadlineCallback = + PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout); + } + + FillMetaHeaders(RpcStuff->Context, Settings.Server.Database, Settings.CredentialsProvider.get()); + RpcStuff->Stub = PersQueueService::NewStub(RpcStuff->Channel); + RpcStuff->Stream = RpcStuff->Stub->AsyncReadSession(&RpcStuff->Context, RpcStuff->CQ.get(), new TQueueEvent(StreamCreatedHandler)); +} + +NThreading::TFuture<TConsumerCreateResponse> TConsumer::Start(TInstant deadline) noexcept +{ + NThreading::TFuture<TConsumerCreateResponse> ret = StartPromise.GetFuture(); + if (ChannelHolder.ChannelInfo.Initialized()) { + std::weak_ptr<TConsumer> self = shared_from_this(); + PQLib->Subscribe(ChannelHolder.ChannelInfo, + this, + [self, deadline](const auto& f) { + auto sharedSelf = self.lock(); + if (sharedSelf) { + sharedSelf->SetChannel(f.GetValue()); + sharedSelf->DoStart(deadline); + } + }); + } else { + if (RpcStuff->Channel && ProxyCookie) { + DoStart(deadline); + } else { + Destroy("No channel"); + } + } + return ret; +} + +void TConsumer::SetChannel(const TChannelHolder& channel) noexcept { + if (IsDestroyed) + return; + + ChannelHolder = channel; +} + +void TConsumer::SetChannel(const TChannelInfo& channel) noexcept { + if (!channel.Channel || channel.ProxyCookie == 0) { + Destroy("Channel creation error"); + return; + } + + if (IsDestroyed) + return; + Y_VERIFY(!RpcStuff->Channel); + RpcStuff->Channel = channel.Channel; + ProxyCookie = channel.ProxyCookie; +} + + +TConsumer::~TConsumer() noexcept { + Destroy(); +} + + +/************************************************read*****************************************************/ + +void TConsumer::OrderRead() noexcept { + if (IsDeadPromise.HasValue()) { + return; + } + + if (StartPromise.Initialized()) { + return; + } + + if (MemoryUsage >= Settings.MaxMemoryUsage || ReadsOrdered >= Settings.MaxInflyRequests) { + return; + } + if (Settings.MaxUncommittedCount > 0 && UncommittedCount >= Settings.MaxUncommittedCount) { + return; + } + if (Settings.MaxUncommittedSize > 0 && UncommittedSize >= Settings.MaxUncommittedSize) { + return; + } + + + if (IsDeadPromise.HasValue()) { + return; + } else { + while (ReadsOrdered < Settings.MaxInflyRequests && MemoryUsage + ((ui64)ReadsOrdered) * EstimateReadSize <= Settings.MaxMemoryUsage) { + TReadRequest request; + Settings.CredentialsProvider->FillAuthInfo(request.MutableCredentials()); + request.MutableRead()->SetMaxCount(Settings.MaxCount); + request.MutableRead()->SetMaxSize(Settings.MaxSize); + request.MutableRead()->SetPartitionsAtOnce(Settings.PartsAtOnce); + request.MutableRead()->SetMaxTimeLagMs(Settings.MaxTimeLagMs); + request.MutableRead()->SetReadTimestampMs(Settings.ReadTimestampMs); + Requests.emplace_back(std::move(request)); + ++ReadsOrdered; + DEBUG_LOG("read ordered memusage " << MemoryUsage << " infly " << ReadsOrdered << " maxmemusage " + << Settings.MaxMemoryUsage, "", SessionId); + } + + DoWrite(); + } +} + +void TConsumer::OnWriteDone() { + WriteInflight = false; + DoWrite(); +} + +void TConsumer::DoWrite() { + if (IsDestroyed) { + return; + } + + if (WriteInflight || Requests.empty()) + return; + + WriteInflight = true; + TReadRequest req = Requests.front(); + Requests.pop_front(); + DEBUG_LOG("sending request " << req.GetRequestCase(), "", SessionId); + RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); +} + + +void TConsumer::OnStreamCreated() { + if (IsDestroyed) { + return; + } + + TReadRequest req; + Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials()); + + TReadRequest::TInit* const init = req.MutableInit(); + init->SetClientId(Settings.ClientId); + for (const auto& t : Settings.Topics) { + init->AddTopics(t); + } + init->SetReadOnlyLocal(!Settings.ReadMirroredPartitions); + init->SetClientsideLocksAllowed(Settings.UseLockSession); + init->SetProxyCookie(ProxyCookie); + init->SetProtocolVersion(TReadRequest::ReadParamsInInit); + init->SetBalancePartitionRightNow(Settings.BalanceRightNow); + init->SetCommitsDisabled(Settings.CommitsDisabled); + + for (auto g : Settings.PartitionGroups) { + init->AddPartitionGroups(g); + } + + // Read settings + init->SetMaxReadMessagesCount(Settings.MaxCount); + init->SetMaxReadSize(Settings.MaxSize); + init->SetMaxReadPartitionsCount(Settings.PartsAtOnce); + init->SetMaxTimeLagMs(Settings.MaxTimeLagMs); + init->SetReadTimestampMs(Settings.ReadTimestampMs); + + WriteInflight = true; + RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); + + RpcStuff->Response.Clear(); + RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); +} + +void TConsumer::OnReadDone() { + if (RpcStuff->Response.HasError()) { + Destroy(RpcStuff->Response.GetError()); + return; + } + + if (IsDeadPromise.HasValue()) + return; + if (StartPromise.Initialized()) { //init response + NThreading::TPromise<TConsumerCreateResponse> tmp; + tmp.Swap(StartPromise); + + Y_VERIFY(RpcStuff->Response.HasInit()); + auto res(std::move(RpcStuff->Response)); + RpcStuff->Response.Clear(); + RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); + + SessionId = res.GetInit().GetSessionId(); + DEBUG_LOG("read stream created", "", SessionId); + + tmp.SetValue(TConsumerCreateResponse{std::move(res)}); + + OrderRead(); + + return; + } + if (RpcStuff->Response.HasBatchedData()) { + ConvertToOldBatch(RpcStuff->Response); // LOGBROKER-3173 + } + if (RpcStuff->Response.HasData()) { //read response + NThreading::TPromise<TConsumerMessage> p(NThreading::NewPromise<TConsumerMessage>()); + MessageResponses.push_back(p.GetFuture()); + const ui32 sz = RpcStuff->Response.ByteSize(); + MemoryUsage += sz; + Y_VERIFY(ReadsOrdered); + --ReadsOrdered; + + ui32 cnt = 0; + for (ui32 i = 0; i < RpcStuff->Response.GetData().MessageBatchSize(); ++i) { + cnt += RpcStuff->Response.GetData().GetMessageBatch(i).MessageSize(); + } + if (!Settings.CommitsDisabled) { + ReadInfo.push_back({RpcStuff->Response.GetData().GetCookie(), {cnt, sz}}); + UncommittedSize += sz; + UncommittedCount += cnt; + } + + DEBUG_LOG("read done memusage " << MemoryUsage << " infly " << ReadsOrdered << " size " << sz + << " ucs " << UncommittedSize << " ucc " << UncommittedCount, "", SessionId); + + auto res(std::move(RpcStuff->Response)); + + RpcStuff->Response.Clear(); + RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); + + OrderRead(); + + Y_VERIFY(MemoryUsage >= sz); + EstimateReadSize = res.ByteSize(); + OrderRead(); + + p.SetValue(TConsumerMessage{std::move(res)}); + } else if (RpcStuff->Response.HasLock() || RpcStuff->Response.HasRelease() || RpcStuff->Response.HasCommit() || RpcStuff->Response.HasPartitionStatus()) { //lock/release/commit response + if (!RpcStuff->Response.HasLock()) { + INFO_LOG("got from server " << RpcStuff->Response, "", SessionId); + if (RpcStuff->Response.HasCommit() && !Settings.CommitsDisabled) { + for (ui32 i = 0; i < RpcStuff->Response.GetCommit().CookieSize(); ++i) { + ui64 cookie = RpcStuff->Response.GetCommit().GetCookie(i); + Y_VERIFY(!ReadInfo.empty() && ReadInfo.front().first == cookie); + Y_VERIFY(UncommittedCount >= ReadInfo.front().second.first); + Y_VERIFY(UncommittedSize >= ReadInfo.front().second.second); + UncommittedCount -= ReadInfo.front().second.first; + UncommittedSize -= ReadInfo.front().second.second; + ReadInfo.pop_front(); + } + } + MessageResponses.push_back(NThreading::MakeFuture<TConsumerMessage>(TConsumerMessage(std::move(RpcStuff->Response)))); + OrderRead(); + } else { + const auto& p = RpcStuff->Response.GetLock(); + NThreading::TPromise<TLockInfo> promise(NThreading::NewPromise<TLockInfo>()); + //TODO: add subscribe + auto f = promise.GetFuture(); + std::weak_ptr<TConsumer> ptr = shared_from_this(); + + //will take ptr for as long as promises are not set + PQLib->Subscribe(f, + this, + [ptr, p](const auto& f) { + auto consumer = ptr.lock(); + if (consumer) { + consumer->Lock(p.GetTopic(), p.GetPartition(), p.GetGeneration(), f.GetValue().ReadOffset, f.GetValue().CommitOffset, + f.GetValue().VerifyReadOffset); + } + }); + + INFO_LOG("got LOCK from server " << p, "", SessionId); + MessageResponses.push_back(NThreading::MakeFuture<TConsumerMessage>(TConsumerMessage(std::move(RpcStuff->Response), std::move(promise)))); + } + + RpcStuff->Response.Clear(); + RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); + } else { + Y_FAIL("unsupported response %s", RpcStuff->Response.DebugString().c_str()); + } + + ProcessResponses(); +} + + +/******************************************commit****************************************************/ +void TConsumer::Commit(const TVector<ui64>& cookies) noexcept { + auto sortedCookies = cookies; + std::sort(sortedCookies.begin(), sortedCookies.end()); + + TReadRequest request; + for (const auto& cookie : sortedCookies) { + request.MutableCommit()->AddCookie(cookie); + } + + DEBUG_LOG("sending COMMIT to server " << request.GetCommit(), "", SessionId); + if (IsDestroyed || StartPromise.Initialized()) { + ERR_LOG("trying to commit " << (IsDestroyed ? "after destroy" : "before start"), "", SessionId); + return; + } + Requests.emplace_back(std::move(request)); + DoWrite(); +} + + +void TConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { + TReadRequest request; + auto status = request.MutableStatus(); + status->SetTopic(topic); + status->SetPartition(partition); + status->SetGeneration(generation); + + DEBUG_LOG("sending GET_STATUS to server " << request.GetStatus(), "", SessionId); + if (IsDestroyed || StartPromise.Initialized()) { + ERR_LOG("trying to get status " << (IsDestroyed ? "after destroy" : "before start"), "", SessionId); + return; + } + Requests.emplace_back(std::move(request)); + DoWrite(); +} + +void TConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { + if (IsDeadPromise.HasValue()) { + TReadResponse res; + res.MutableError()->SetDescription("consumer is dead"); + res.MutableError()->SetCode(NErrorCode::ERROR); + promise.SetValue(TConsumerMessage{std::move(res)}); + return; + } + + if (StartPromise.Initialized()) { + TReadResponse res; + res.MutableError()->SetDescription("consumer is not ready"); + res.MutableError()->SetCode(NErrorCode::ERROR); + promise.SetValue(TConsumerMessage{std::move(res)}); + return; + } + + MessagePromises.push_back(promise); + ProcessResponses(); //if alredy have an answer +} + + +void TConsumer::Lock(const TString& topic, const ui32 partition, const ui64 generation, const ui64 readOffset, + const ui64 commitOffset, const bool verifyReadOffset) noexcept { + Y_VERIFY(Settings.UseLockSession); + + TReadRequest request; + request.MutableStartRead()->SetTopic(topic); + request.MutableStartRead()->SetPartition(partition); + request.MutableStartRead()->SetReadOffset(readOffset); + request.MutableStartRead()->SetCommitOffset(commitOffset); + request.MutableStartRead()->SetVerifyReadOffset(verifyReadOffset); + request.MutableStartRead()->SetGeneration(generation); + INFO_LOG("sending START_READ to server " << request.GetStartRead(), "", SessionId); + + Settings.CredentialsProvider->FillAuthInfo(request.MutableCredentials()); + + if (IsDestroyed) + return; + Requests.emplace_back(std::move(request)); + DoWrite(); +} + + +void TConsumer::ProcessResponses() { + while(true) { + if (MessagePromises.empty() || MessageResponses.empty() || !MessageResponses.front().HasValue()) + break; + auto p(std::move(MessagePromises.front())); + MessagePromises.pop_front(); + auto res(MessageResponses.front().ExtractValue()); + MessageResponses.pop_front(); + if (res.Type == EMT_DATA) { + ui32 sz = res.Response.ByteSize(); + Y_VERIFY(MemoryUsage >= sz); + MemoryUsage -= sz; + } + + if (res.Type == EMT_DATA) { + OrderRead(); + } + + p.SetValue(TConsumerMessage{std::move(res)}); + } +} + +/***********************************************************************************/ + +NThreading::TFuture<TError> TConsumer::IsDead() noexcept { + return IsDeadPromise.GetFuture(); +} + +void TConsumer::Destroy(const TString& description) { + TError error; + error.SetDescription(description); + error.SetCode(NErrorCode::ERROR); + Destroy(error); +} + +void TConsumer::Destroy(const TError& error) { + if (!IsDestroying) { + INFO_LOG("error: " << error, "", SessionId); + } + + if (IsDestroyed) + return; + IsDestroyed = true; + + IsDeadPromise.SetValue(error); + + MessageResponses.clear(); + + NThreading::TFuture<TChannelInfo> tmp; + tmp.Swap(ChannelHolder.ChannelInfo); + ChannelHolder.ChannelPtr = nullptr; + + Error = error; + if (StartPromise.Initialized()) { + NThreading::TPromise<TConsumerCreateResponse> tmp; + tmp.Swap(StartPromise); + TReadResponse res; + res.MutableError()->CopyFrom(Error); + tmp.SetValue(TConsumerCreateResponse{std::move(res)}); + } + + while (!MessagePromises.empty()) { + auto p = MessagePromises.front(); + MessagePromises.pop_front(); + TReadResponse res; + res.MutableError()->CopyFrom(Error); + p.SetValue(TConsumerMessage{std::move(res)}); + } + + StreamCreatedHandler.Reset(); + ReadDoneHandler.Reset(); + WriteDoneHandler.Reset(); + + DestroyPQLibRef(); +} + +void TConsumer::OnStartTimeout() { + if (IsDestroyed) { + return; + } + + StartDeadlineCallback = nullptr; + if (!StartPromise.Initialized()) { + // already replied with successful start + return; + } + + TError error; + error.SetDescription("Start timeout"); + error.SetCode(NErrorCode::CREATE_TIMEOUT); + Destroy(error); +} + +void TConsumer::Cancel() { + IsDestroying = true; + RpcStuff->Context.TryCancel(); + ChannelHolder.ChannelPtr = nullptr; //if waiting for channel creation + + Destroy(GetCancelReason()); +} + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h new file mode 100644 index 0000000000..f2f6f81e71 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer.h @@ -0,0 +1,121 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include "channel.h" +#include "scheduler.h" +#include "iconsumer_p.h" +#include "internals.h" +#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> + +#include <library/cpp/threading/future/future.h> + +#include <deque> +#include <memory> + +namespace NPersQueue { + +class TConsumer: public IConsumerImpl, public std::enable_shared_from_this<TConsumer> { +public: + friend class TPQLibPrivate; + + NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override; + + void Commit(const TVector<ui64>& cookies) noexcept override; + + using IConsumerImpl::GetNextMessage; + void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; + + void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; + + NThreading::TFuture<TError> IsDead() noexcept override; + + TConsumer(const TConsumerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, + NThreading::TPromise<TConsumerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept; + + void Init() override; + + ~TConsumer() noexcept; + + void Destroy() noexcept; + void SetChannel(const TChannelHolder& channel) noexcept; + void SetChannel(const TChannelInfo& channel) noexcept; + + void Lock(const TString& topic, const ui32 partition, const ui64 generation, const ui64 readOffset, const ui64 commitOffset, const bool verifyReadOffset) noexcept; + void OrderRead() noexcept; + + void Cancel() override; + +public: + using TStream = grpc::ClientAsyncReaderWriterInterface<TReadRequest, TReadResponse>; + + // objects that must live after destruction of producer untill all the callbacks arrive at CompletionQueue + struct TRpcStuff: public TAtomicRefCount<TRpcStuff> { + TReadResponse Response; + std::shared_ptr<grpc::CompletionQueue> CQ; + std::shared_ptr<grpc::Channel> Channel; + std::unique_ptr<PersQueueService::Stub> Stub; + grpc::ClientContext Context; + std::unique_ptr<TStream> Stream; + }; + +protected: + friend class TConsumerStreamCreated; + friend class TConsumerReadDone; + friend class TConsumerWriteDone; + friend class TConsumerDestroyHandler; + + IHandlerPtr StreamCreatedHandler; + IHandlerPtr ReadDoneHandler; + IHandlerPtr WriteDoneHandler; + + void ProcessResponses(); + void Destroy(const TError& reason); + void Destroy(const TString& description); // the same but with Code=ERROR + void OnStreamCreated(); + void OnReadDone(); + void OnWriteDone(); + + void DoWrite(); + void DoStart(TInstant deadline); + void OnStartTimeout(); + + TIntrusivePtr<TRpcStuff> RpcStuff; + + TChannelHolder ChannelHolder; + TConsumerSettings Settings; + + TString SessionId; + + NThreading::TPromise<TConsumerCreateResponse> StartPromise; + NThreading::TPromise<TError> IsDeadPromise; + std::deque<NThreading::TPromise<TConsumerMessage>> MessagePromises; + std::deque<NThreading::TFuture<TConsumerMessage>> MessageResponses; + + std::deque<TReadRequest> Requests; + + + std::deque<std::pair<ui64, std::pair<ui32, ui64>>> ReadInfo; //cookie -> (count, size) + ui32 UncommittedCount = 0; + ui64 UncommittedSize = 0; + + ui64 MemoryUsage = 0; + ui32 ReadsOrdered = 0; + + ui64 EstimateReadSize = 0; + + bool WriteInflight; + + ui64 ProxyCookie = 0; + + TIntrusivePtr<ILogger> Logger; + + TError Error; + + bool IsDestroyed; + bool IsDestroying; + + TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp new file mode 100644 index 0000000000..dc09bd2ef1 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/consumer_ut.cpp @@ -0,0 +1,90 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NPersQueue { +Y_UNIT_TEST_SUITE(TConsumerTest) { + Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) { + // Test that consumer doesn't hang on till shutdown + TPQLib lib; + TConsumerSettings settings; + settings.Server = TServerSetting{"localhost"}; + settings.Topics.push_back("topic"); + settings.ClientId = "client"; + lib.CreateConsumer(settings, {}, false); + } + + Y_UNIT_TEST(StartTimeout) { + if (GrpcV1EnabledByDefault()) { + return; + } + + TTestServer testServer; + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic", 1); + testServer.WaitInit("topic"); + // Test that consumer doesn't hang on till shutdown + TConsumerSettings settings; + settings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + settings.Topics.push_back("topic"); + settings.ClientId = "client"; + auto consumer = testServer.PQLib->CreateConsumer(settings, {}, false); + auto startResult = consumer->Start(TInstant::Now()); + UNIT_ASSERT_EQUAL_C(startResult.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, startResult.GetValueSync().Response); + + DestroyAndWait(consumer); + } + + TProducerSettings MakeProducerSettings(const TTestServer& testServer) { + TProducerSettings producerSettings; + producerSettings.ReconnectOnFailure = false; + producerSettings.Topic = "topic1"; + producerSettings.SourceId = "123"; + producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + producerSettings.Codec = ECodec::LZOP; + return producerSettings; + } + + TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { + TConsumerSettings settings; + settings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + settings.Topics.emplace_back("topic1"); + settings.ClientId = "user"; + return settings; + } + + Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + + const size_t partitions = 1; + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + testServer.WaitInit("topic1"); + + + auto consumer = testServer.PQLib->CreateConsumer(MakeConsumerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false); + UNIT_ASSERT(!consumer->Start().GetValueSync().Response.HasError()); + auto isDead = consumer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + auto msg1 = consumer->GetNextMessage(); + auto msg2 = consumer->GetNextMessage(); + + testServer.PQLib = nullptr; + Cerr << "PQLib destroyed" << Endl; + + UNIT_ASSERT(msg1.HasValue()); + UNIT_ASSERT(msg2.HasValue()); + + UNIT_ASSERT(msg1.GetValue().Response.HasError()); + UNIT_ASSERT(msg2.GetValue().Response.HasError()); + + auto msg3 = consumer->GetNextMessage(); + UNIT_ASSERT(msg3.HasValue()); + UNIT_ASSERT(msg3.GetValue().Response.HasError()); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp new file mode 100644 index 0000000000..e3de1ba3b5 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/credentials_provider.cpp @@ -0,0 +1,349 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/credentials_provider.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include "internals.h" + +#include <library/cpp/json/json_reader.h> +#include <library/cpp/http/simple/http_client.h> +#include <library/cpp/logger/priority.h> +#include <library/cpp/tvmauth/client/facade.h> + +#include <util/generic/ptr.h> +#include <util/string/builder.h> +#include <util/system/env.h> +#include <util/system/mutex.h> +#include <library/cpp/string_utils/quote/quote.h> +#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h> +#include <kikimr/public/sdk/cpp/client/iam/iam.h> + +namespace NPersQueue { + +TString GetToken(ICredentialsProvider* credentials) { + NPersQueue::TCredentials auth; + credentials->FillAuthInfo(&auth); + TString token; + switch (auth.GetCredentialsCase()) { + case NPersQueue::TCredentials::kTvmServiceTicket: + token = auth.GetTvmServiceTicket(); + break; + case NPersQueue::TCredentials::kOauthToken: + token = auth.GetOauthToken(); + break; + case NPersQueue::TCredentials::CREDENTIALS_NOT_SET: + break; + default: + Y_VERIFY(true, "Unknown Credentials case."); + } + return token; +} + +const TString& GetTvmPqServiceName() { + static const TString PQ_SERVICE_NAME = "pq"; + return PQ_SERVICE_NAME; +} + +class TInsecureCredentialsProvider : public ICredentialsProvider { + void FillAuthInfo(NPersQueue::TCredentials*) const override {} +}; + +class TLogBridge: public NTvmAuth::ILogger { +public: + TLogBridge(TIntrusivePtr<NPersQueue::ILogger> logger = nullptr) + : Logger(logger) + {} + + void Log(int lvl, const TString& msg) override { + if (Logger) { + Logger->Log(msg, "", "", lvl); + } + } + +private: + TIntrusivePtr<NPersQueue::ILogger> Logger; +}; + +class TTVMCredentialsProvider : public ICredentialsProvider { +public: + TTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr) + : Alias(alias) + { + if (!settings.GetDstAliases().contains(Alias)) { + ythrow yexception() << "alias for `" << Alias << "` must be set"; + } + Logger = MakeIntrusive<TLogBridge>(std::move(logger)); + TvmClient = std::make_shared<NTvmAuth::TTvmClient>(settings, Logger); + } + + TTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr) + : TTVMCredentialsProvider(CreateSettings(secret, srcClientId, dstClientId, alias), alias, logger) + {} + + TTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, const TString& alias, TIntrusivePtr<ILogger> logger = nullptr) + : TvmClient(tvmClient) + , Alias(alias) + , Logger(MakeIntrusive<TLogBridge>(std::move(logger))) + {} + + void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { + try { + auto status = TvmClient->GetStatus(); + if (status == NTvmAuth::TClientStatus::Ok) { + authInfo->SetTvmServiceTicket(TvmClient->GetServiceTicketFor(Alias)); + } else { + Logger->Error(TStringBuilder() << "Can't get ticket: " << status.GetLastError() << "\n"); + } + } catch (...) { + Logger->Error(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n"); + } + } + +private: + std::shared_ptr<NTvmAuth::TTvmClient> TvmClient; + TString Alias; + NTvmAuth::TLoggerPtr Logger; + + static NTvmAuth::NTvmApi::TClientSettings CreateSettings(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, const TString& alias) { + NTvmAuth::NTvmApi::TClientSettings settings; + settings.SetSelfTvmId(srcClientId); + settings.EnableServiceTicketsFetchOptions(secret, {{alias, dstClientId}}); + return settings; + } +}; + +class TTVMQloudCredentialsProvider : public ICredentialsProvider { +public: + TTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, const TDuration& refreshPeriod, TIntrusivePtr<ILogger> logger, ui32 port) + : HttpClient(TSimpleHttpClient("localhost", port)) + , Request(TStringBuilder() << "/tvm/tickets?src=" << CGIEscapeRet(srcAlias) << "&dsts=" << CGIEscapeRet(dstAlias)) + , DstAlias(dstAlias) + , LastTicketUpdate(TInstant::Zero()) + , RefreshPeriod(refreshPeriod) + , Logger(std::move(logger)) + , DstId(0) + { + GetTicket(); + } + + TTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, const TDuration& refreshPeriod, TIntrusivePtr<ILogger> logger, ui32 port) + : HttpClient(TSimpleHttpClient("localhost", port)) + , Request(TStringBuilder() << "/tvm/tickets?src=" << srcId << "&dsts=" << dstId) + , LastTicketUpdate(TInstant::Zero()) + , RefreshPeriod(refreshPeriod) + , Logger(std::move(logger)) + , DstId(dstId) + { + GetTicket(); + } + + void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { + if (TInstant::Now() > LastTicketUpdate + RefreshPeriod) { + GetTicket(); + } + + authInfo->SetTvmServiceTicket(Ticket); + } + +private: + TSimpleHttpClient HttpClient; + TString Request; + TString DstAlias; + mutable TString Ticket; + mutable TInstant LastTicketUpdate; + TDuration RefreshPeriod; + TIntrusivePtr<ILogger> Logger; + ui32 DstId; + + void GetTicket() const { + try { + TStringStream out; + TSimpleHttpClient::THeaders headers; + headers["Authorization"] = GetEnv("QLOUD_TVM_TOKEN"); + HttpClient.DoGet(Request, &out, headers); + NJson::TJsonValue resp; + NJson::ReadJsonTree(&out, &resp, true); + TString localDstAlias = DstAlias; + Y_VERIFY(!DstAlias.empty() || DstId != 0); + Y_VERIFY(resp.GetMap().size() == 1); + if (!localDstAlias.empty()) { + if (!resp.Has(localDstAlias)) { + ythrow yexception() << "Result doesn't contain dstAlias `" << localDstAlias << "`"; + } + } + else { + for (const auto &[k, v] : resp.GetMap()) { + if (!v.Has("tvm_id") || v["tvm_id"].GetIntegerSafe() != DstId) + ythrow yexception() << "Result doesn't contain dstId `" << DstId << "`"; + localDstAlias = k; + } + } + TString ticket = resp[localDstAlias]["ticket"].GetStringSafe(); + if (ticket.empty()) { + ythrow yexception() << "Got empty ticket"; + } + Ticket = ticket; + LastTicketUpdate = TInstant::Now(); + } catch (...) { + if (Logger) { + Logger->Log(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n", "", "", TLOG_ERR); + } + } + } +}; + +class TOAuthCredentialsProvider : public ICredentialsProvider { +public: + TOAuthCredentialsProvider(const TString& token) + : Token(token) + {} + + void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { + authInfo->SetOauthToken(Token); + } + +private: + TString Token; +}; + +class TTVMCredentialsForwarder : public ITVMCredentialsForwarder { +public: + TTVMCredentialsForwarder() = default; + + void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { + TGuard<TSpinLock> guard(Lock); + if (!Ticket.empty()) + authInfo->SetTvmServiceTicket(Ticket); + } + + void SetTVMServiceTicket(const TString& ticket) override { + TGuard<TSpinLock> guard(Lock); + Ticket = ticket; + } + + +private: + TString Ticket; + TSpinLock Lock; +}; + +class IIAMCredentialsProviderWrapper : public ICredentialsProvider { +public: + IIAMCredentialsProviderWrapper(std::shared_ptr<NYdb::ICredentialsProviderFactory> factory, TIntrusivePtr<ILogger> logger = nullptr) + : Provider(factory->CreateProvider()) + , Logger(std::move(logger)) + , Lock() + { + } + + void FillAuthInfo(NPersQueue::TCredentials* authInfo) const override { + TString ticket; + try { + with_lock(Lock) { + ticket = Provider->GetAuthInfo(); + } + } catch (...) { + if (Logger) { + Logger->Log(TStringBuilder() << "Can't get ticket: " << CurrentExceptionMessage() << "\n", "", "", TLOG_ERR); + } + return; + } + + authInfo->SetTvmServiceTicket(ticket); + } + +private: + std::shared_ptr<NYdb::ICredentialsProvider> Provider; + TIntrusivePtr<ILogger> Logger; + TMutex Lock; +}; + +class TIAMCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper { +public: + TIAMCredentialsProviderWrapper(TIntrusivePtr<ILogger> logger = nullptr) + : IIAMCredentialsProviderWrapper( + NYdb::CreateIamCredentialsProviderFactory(), + std::move(logger) + ) + { + } +}; + +class TIAMJwtFileCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper { +public: + TIAMJwtFileCredentialsProviderWrapper( + const TString& jwtKeyFilename, const TString& endpoint, const TDuration& refreshPeriod, + const TDuration& requestTimeout, TIntrusivePtr<ILogger> logger = nullptr + ) + : IIAMCredentialsProviderWrapper( + NYdb::CreateIamJwtFileCredentialsProviderFactory( + {{endpoint, refreshPeriod, requestTimeout}, jwtKeyFilename}), + std::move(logger) + ) + {} +}; + +class TIAMJwtParamsCredentialsProviderWrapper : public IIAMCredentialsProviderWrapper { +public: + TIAMJwtParamsCredentialsProviderWrapper( + const TString& jwtParams, const TString& endpoint, const TDuration& refreshPeriod, + const TDuration& requestTimeout, TIntrusivePtr<ILogger> logger = nullptr + ) + : IIAMCredentialsProviderWrapper( + NYdb::CreateIamJwtParamsCredentialsProviderFactory( + {{endpoint, refreshPeriod, requestTimeout}, jwtParams}), + std::move(logger) + ) + {} +}; + +std::shared_ptr<ICredentialsProvider> CreateInsecureCredentialsProvider() { + return std::make_shared<TInsecureCredentialsProvider>(); +} + +std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const TString& secret, const ui32 srcClientId, const ui32 dstClientId, TIntrusivePtr<ILogger> logger) { + return std::make_shared<TTVMCredentialsProvider>(secret, srcClientId, dstClientId, GetTvmPqServiceName(), std::move(logger)); +} + +std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(const NTvmAuth::NTvmApi::TClientSettings& settings, TIntrusivePtr<ILogger> logger, const TString& alias) { + return std::make_shared<TTVMCredentialsProvider>(settings, alias, std::move(logger)); +} + +std::shared_ptr<ICredentialsProvider> CreateTVMCredentialsProvider(std::shared_ptr<NTvmAuth::TTvmClient> tvmClient, TIntrusivePtr<ILogger> logger, const TString& alias) { + return std::make_shared<TTVMCredentialsProvider>(tvmClient, alias, std::move(logger)); +} + +std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const TString& srcAlias, const TString& dstAlias, TIntrusivePtr<ILogger> logger, const TDuration refreshPeriod, ui32 port) { + return std::make_shared<TTVMQloudCredentialsProvider>(srcAlias, dstAlias, refreshPeriod, std::move(logger), port); +} + +std::shared_ptr<ICredentialsProvider> CreateTVMQloudCredentialsProvider(const ui32 srcId, const ui32 dstId, TIntrusivePtr<ILogger> logger, const TDuration refreshPeriod, ui32 port) { + return std::make_shared<TTVMQloudCredentialsProvider>(srcId, dstId, refreshPeriod, std::move(logger), port); +} + +std::shared_ptr<ICredentialsProvider> CreateOAuthCredentialsProvider(const TString& token) { + return std::make_shared<TOAuthCredentialsProvider>(token); +} + +std::shared_ptr<ITVMCredentialsForwarder> CreateTVMCredentialsForwarder() { + return std::make_shared<TTVMCredentialsForwarder>(); +} + +std::shared_ptr<ICredentialsProvider> CreateIAMCredentialsForwarder(TIntrusivePtr<ILogger> logger) { + return std::make_shared<TIAMCredentialsProviderWrapper>(logger); +} + +std::shared_ptr<ICredentialsProvider> CreateIAMJwtFileCredentialsForwarder( + const TString& jwtKeyFilename, TIntrusivePtr<ILogger> logger, + const TString& endpoint, const TDuration& refreshPeriod, const TDuration& requestTimeout +) { + return std::make_shared<TIAMJwtFileCredentialsProviderWrapper>(jwtKeyFilename, endpoint, + refreshPeriod, requestTimeout, logger); +} + +std::shared_ptr<ICredentialsProvider> CreateIAMJwtParamsCredentialsForwarder( + const TString& jwtParams, TIntrusivePtr<ILogger> logger, + const TString& endpoint, const TDuration& refreshPeriod, const TDuration& requestTimeout +) { + return std::make_shared<TIAMJwtParamsCredentialsProviderWrapper>(jwtParams, endpoint, + refreshPeriod, requestTimeout, logger); +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp new file mode 100644 index 0000000000..0fbf62cf02 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.cpp @@ -0,0 +1,290 @@ +#include "decompressing_consumer.h" +#include "persqueue_p.h" + +#include <library/cpp/streams/lzop/lzop.h> +#include <library/cpp/streams/zstd/zstd.h> + +#include <util/stream/mem.h> +#include <util/stream/zlib.h> + +#include <atomic> + +namespace NPersQueue { + +TDecompressingConsumer::TDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) + : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) + , Logger(std::move(logger)) + , Subconsumer(std::move(subconsumer)) + , Settings(settings) +{ +} + +void TDecompressingConsumer::Init() { + DestroyedFuture = WaitExceptionOrAll(DestroyedPromise.GetFuture(), Subconsumer->Destroyed()); +} + +TDecompressingConsumer::~TDecompressingConsumer() { + DestroyQueue("Destructor called"); +} + +template <class T> +void TDecompressingConsumer::SubscribeForQueueProcessing(NThreading::TFuture<T>& future) { + const void* queueTag = this; + std::weak_ptr<TDecompressingConsumer> self = shared_from_this(); + auto signalProcessQueue = [queueTag, self, pqLib = PQLib.Get()](const auto&) mutable { + SignalProcessQueue(queueTag, std::move(self), pqLib); + }; + future.Subscribe(signalProcessQueue); +} + +NThreading::TFuture<TConsumerCreateResponse> TDecompressingConsumer::Start(TInstant deadline) noexcept { + auto ret = Subconsumer->Start(deadline); + SubscribeForQueueProcessing(ret); // when subconsumer starts, we will read ahead some messages + + // subscribe to death + std::weak_ptr<TDecompressingConsumer> self = shared_from_this(); + auto isDeadHandler = [self](const auto& error) { + auto selfShared = self.lock(); + if (selfShared && !selfShared->IsDestroyed) { + selfShared->DestroyQueue(error.GetValue()); + } + }; + PQLib->Subscribe(Subconsumer->IsDead(), this, isDeadHandler); + + return ret; +} + +NThreading::TFuture<TError> TDecompressingConsumer::IsDead() noexcept { + return IsDeadPromise.GetFuture(); +} + +void TDecompressingConsumer::Commit(const TVector<ui64>& cookies) noexcept { + if (IsDestroyed) { + ERR_LOG("Attempt to commit to dead consumer.", "", ""); + return; + } + return Subconsumer->Commit(cookies); +} + +void TDecompressingConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { + if (IsDestroyed) { + ERR_LOG("Attempt to request status from dead consumer.", "", ""); + return; + } + return Subconsumer->RequestPartitionStatus(topic, partition, generation); +} + + +NThreading::TFuture<void> TDecompressingConsumer::Destroyed() noexcept { + return DestroyedFuture; +} + +void TDecompressingConsumer::AddNewGetNextMessageRequest(NThreading::TPromise<TConsumerMessage>& promise) { + Queue.emplace_back(promise); + Queue.back().Future = Subconsumer->GetNextMessage(); + SubscribeForQueueProcessing(Queue.back().Future); +} + +void TDecompressingConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { + if (IsDestroyed) { + TReadResponse response; + auto* err = response.MutableError(); + err->SetCode(NErrorCode::ERROR); + err->SetDescription("Consumer is dead."); + promise.SetValue(TConsumerMessage(std::move(response))); + return; + } + AddNewGetNextMessageRequest(promise); +} + +void TDecompressingConsumer::ProcessQueue() { + try { + // stage 1: wait subconsumer answer and run decompression tasks + for (TReadRequestInfo& request : Queue) { + if (!request.Data && request.Future.HasValue()) { + request.Data.ConstructInPlace(request.Future.ExtractValue()); + if (request.Data->Type == EMT_DATA) { + RequestDecompressing(request); + } + } + } + + // stage 2: answer ready tasks to client + while (!Queue.empty()) { + TReadRequestInfo& front = Queue.front(); + if (!front.Data || front.Data->Type == EMT_DATA && !front.AllDecompressing.HasValue()) { + break; + } + if (front.Data->Type == EMT_DATA) { + CopyDataToAnswer(front); + } + + front.Promise.SetValue(TConsumerMessage(std::move(*front.Data))); + Queue.pop_front(); + } + } catch (const std::exception&) { + DestroyQueue(TStringBuilder() << "Failed to decompress data: " << CurrentExceptionMessage()); + } +} + +void TDecompressingConsumer::SignalProcessQueue(const void* queueTag, std::weak_ptr<TDecompressingConsumer> self, TPQLibPrivate* pqLib) { + auto processQueue = [self] { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->ProcessQueue(); + } + }; + Y_VERIFY(pqLib->GetQueuePool().GetQueue(queueTag).AddFunc(processQueue)); +} + +namespace { + +struct TWaitAll: public TAtomicRefCount<TWaitAll> { + TWaitAll(size_t count) + : Count(count) + { + } + + void OnResultReady() { + if (--Count == 0) { + Promise.SetValue(); + } + } + + std::atomic<size_t> Count; + NThreading::TPromise<void> Promise = NThreading::NewPromise<void>(); +}; + +THolder<IInputStream> CreateDecompressor(const ECodec codec, IInputStream* origin) { + THolder<IInputStream> result; + if (codec == ECodec::GZIP) { + result.Reset(new TZLibDecompress(origin)); + } else if (codec == ECodec::LZOP) { + result.Reset(new TLzopDecompress(origin)); + } else if (codec == ECodec::ZSTD) { + result.Reset(new TZstdDecompress(origin)); + } + return result; +} + +} // namespace + +void TDecompressingConsumer::RequestDecompressing(TReadRequestInfo& request) { + size_t futuresCount = 0; + const TReadResponse::TData& data = request.Data->Response.GetData(); + request.BatchFutures.resize(data.MessageBatchSize()); + for (size_t i = 0; i < data.MessageBatchSize(); ++i) { + const TReadResponse::TData::TMessageBatch& batch = data.GetMessageBatch(i); + request.BatchFutures.reserve(batch.MessageSize()); + for (const TReadResponse::TData::TMessage& message : batch.GetMessage()) { + if (message.GetMeta().GetCodec() != ECodec::RAW) { + ++futuresCount; + request.BatchFutures[i].push_back(RequestDecompressing(message)); + } + } + } + + if (futuresCount == 0) { + request.AllDecompressing = NThreading::MakeFuture(); // done + return; + } else { + TIntrusivePtr<TWaitAll> waiter = new TWaitAll(futuresCount); + auto handler = [waiter](const auto&) { + waiter->OnResultReady(); + }; + for (auto& batch : request.BatchFutures) { + for (auto& future : batch) { + future.Subscribe(handler); + } + } + request.AllDecompressing = waiter->Promise.GetFuture(); + } + SubscribeForQueueProcessing(request.AllDecompressing); +} + +NThreading::TFuture<TString> TDecompressingConsumer::RequestDecompressing(const TReadResponse::TData::TMessage& message) { + TString data = message.GetData(); + ECodec codec = message.GetMeta().GetCodec(); + NThreading::TPromise<TString> promise = NThreading::NewPromise<TString>(); + auto decompress = [data, codec, promise]() mutable { + Decompress(data, codec, promise); + }; + Y_VERIFY(PQLib->GetCompressionPool().AddFunc(decompress)); + return promise.GetFuture(); +} + +void TDecompressingConsumer::Decompress(const TString& data, ECodec codec, NThreading::TPromise<TString>& promise) { + try { + // TODO: Check if decompression was successfull, i.e. if 'data' is valid byte array compressed with 'codec' + TMemoryInput iss(data.data(), data.size()); + THolder<IInputStream> dec = CreateDecompressor(codec, &iss); + if (!dec) { + ythrow yexception() << "Failed to create decompressor"; + } + + TString result; + TStringOutput oss(result); + TransferData(dec.Get(), &oss); + + promise.SetValue(result); + } catch (...) { + promise.SetException(std::current_exception()); + } +} + +void TDecompressingConsumer::CopyDataToAnswer(TReadRequestInfo& request) { + TReadResponse::TData& data = *request.Data->Response.MutableData(); + Y_VERIFY(request.BatchFutures.size() == data.MessageBatchSize()); + for (size_t i = 0; i < request.BatchFutures.size(); ++i) { + TReadResponse::TData::TMessageBatch& batch = *data.MutableMessageBatch(i); + auto currentMessage = request.BatchFutures[i].begin(); + for (TReadResponse::TData::TMessage& message : *batch.MutableMessage()) { + if (message.GetMeta().GetCodec() != ECodec::RAW) { + Y_VERIFY(currentMessage != request.BatchFutures[i].end()); + try { + message.SetData(currentMessage->GetValue()); + message.MutableMeta()->SetCodec(ECodec::RAW); + } catch (const std::exception& ex) { + if (Settings.SkipBrokenChunks) { + message.SetBrokenPackedData(message.GetData()); + message.SetData(""); + } else { + throw; + } + } + ++currentMessage; + } + } + Y_VERIFY(currentMessage == request.BatchFutures[i].end()); + } +} + +void TDecompressingConsumer::DestroyQueue(const TString& errorMessage) { + TError error; + error.SetDescription(errorMessage); + error.SetCode(NErrorCode::ERROR); + DestroyQueue(error); +} + +void TDecompressingConsumer::DestroyQueue(const TError& error) { + if (IsDestroyed) { + return; + } + + for (TReadRequestInfo& request : Queue) { + TReadResponse response; + *response.MutableError() = error; + request.Promise.SetValue(TConsumerMessage(std::move(response))); + } + Queue.clear(); + IsDeadPromise.SetValue(error); + IsDestroyed = true; + + DestroyPQLibRef(); +} + +void TDecompressingConsumer::Cancel() { + DestroyQueue(GetCancelReason()); +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h new file mode 100644 index 0000000000..c547ef8b3a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer.h @@ -0,0 +1,77 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include "internals.h" +#include "iconsumer_p.h" + +#include <util/generic/maybe.h> + +#include <deque> +#include <memory> +#include <vector> + +namespace NPersQueue { + +class TPQLibPrivate; + +class TDecompressingConsumer: public IConsumerImpl, public std::enable_shared_from_this<TDecompressingConsumer> { +public: + using IConsumerImpl::GetNextMessage; + NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override; + NThreading::TFuture<TError> IsDead() noexcept override; + NThreading::TFuture<void> Destroyed() noexcept override; + void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; + void Commit(const TVector<ui64>& cookies) noexcept override; + void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; + void Init() override; + + ~TDecompressingConsumer(); + TDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); + + void Cancel() override; + +private: + struct TReadRequestInfo { + TReadRequestInfo() = default; + + TReadRequestInfo(const NThreading::TPromise<TConsumerMessage>& promise) + : Promise(promise) + { + } + + NThreading::TPromise<TConsumerMessage> Promise; + NThreading::TFuture<TConsumerMessage> Future; + TMaybe<TConsumerMessage> Data; + + using TMessageFutures = std::vector<NThreading::TFuture<TString>>; + using TBatchFutures = std::vector<TMessageFutures>; + TBatchFutures BatchFutures; + NThreading::TFuture<void> AllDecompressing; + }; + + void ProcessQueue(); + static void SignalProcessQueue(const void* queueTag, std::weak_ptr<TDecompressingConsumer> self, TPQLibPrivate* pqLib); + void DestroyQueue(const TString& errorMessage); + void DestroyQueue(const TError& error); + + template <class T> + void SubscribeForQueueProcessing(NThreading::TFuture<T>& future); + + void RequestDecompressing(TReadRequestInfo& request); + void CopyDataToAnswer(TReadRequestInfo& request); + NThreading::TFuture<TString> RequestDecompressing(const TReadResponse::TData::TMessage& message); + static void Decompress(const TString& data, ECodec codec, NThreading::TPromise<TString>& promise); + void AddNewGetNextMessageRequest(NThreading::TPromise<TConsumerMessage>& promise); + +protected: + TIntrusivePtr<ILogger> Logger; + std::shared_ptr<IConsumerImpl> Subconsumer; + std::deque<TReadRequestInfo> Queue; + TConsumerSettings Settings; + NThreading::TFuture<void> DestroyedFuture; // Subconsumer may be deleted, so store our future here. + NThreading::TPromise<TError> IsDeadPromise = NThreading::NewPromise<TError>(); + bool IsDestroyed = false; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp new file mode 100644 index 0000000000..6c22163e8f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/decompressing_consumer_ut.cpp @@ -0,0 +1,265 @@ +#include "decompressing_consumer.h" +#include "local_caller.h" +#include "persqueue_p.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> + +#include <library/cpp/testing/gmock_in_unittest/gmock.h> +#include <library/cpp/testing/unittest/registar.h> + +using namespace testing; + +namespace NPersQueue { + +class TMockConsumer: public IConsumerImpl { +public: + TMockConsumer() + : IConsumerImpl(nullptr, nullptr) + { + } + + NThreading::TFuture<TConsumerCreateResponse> Start(TInstant) noexcept override { + return NThreading::MakeFuture<TConsumerCreateResponse>(MockStart()); + } + + NThreading::TFuture<TError> IsDead() noexcept override { + return MockIsDead(); + } + + NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept override { + return NThreading::MakeFuture<TConsumerMessage>(MockGetNextMessage()); + } + + void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override { + promise.SetValue(MockGetNextMessage()); + } + + void Commit(const TVector<ui64>&) noexcept override { + } + + void RequestPartitionStatus(const TString&, ui64, ui64) noexcept override { + } + + void Cancel() override { + } + + MOCK_METHOD(TConsumerCreateResponse, MockStart, (), ()); + MOCK_METHOD(NThreading::TFuture<TError>, MockIsDead, (), ()); + MOCK_METHOD(TConsumerMessage, MockGetNextMessage, (), ()); +}; + +template <class TMock = TMockConsumer> +struct TDecompressingConsumerBootstrap { + ~TDecompressingConsumerBootstrap() { + Lib->CancelObjectsAndWait(); + } + + void Create() { + Lib = new TPQLibPrivate(PQLibSettings); + MockConsumer = std::make_shared<TMock>(); + DecompressingConsumer = std::make_shared<TLocalConsumerImplCaller<TDecompressingConsumer>>(MockConsumer, Settings, Lib->GetSelfRefsAreDeadPtr(), Lib, Logger); + Lib->AddToDestroySet(DecompressingConsumer); + } + + void MakeOKStartResponse() { + TReadResponse rresp; + rresp.MutableInit()->SetSessionId("test-session"); + TConsumerCreateResponse resp(std::move(rresp)); + EXPECT_CALL(*MockConsumer, MockStart()) + .WillOnce(Return(resp)); + } + + void ExpectIsDeadCall() { + EXPECT_CALL(*MockConsumer, MockIsDead()) + .WillOnce(Return(DeadPromise.GetFuture())); + } + + void Start() { + MakeOKStartResponse(); + UNIT_ASSERT_STRINGS_EQUAL(DecompressingConsumer->Start().GetValueSync().Response.GetInit().GetSessionId(), "test-session"); + } + + TConsumerSettings Settings; + TPQLibSettings PQLibSettings; + TIntrusivePtr<TCerrLogger> Logger = new TCerrLogger(TLOG_DEBUG); + TIntrusivePtr<TPQLibPrivate> Lib; + std::shared_ptr<TMock> MockConsumer; + std::shared_ptr<IConsumerImpl> DecompressingConsumer; + NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); +}; + +Y_UNIT_TEST_SUITE(TDecompressingConsumerTest) { + Y_UNIT_TEST(DiesOnDeadSubconsumer) { + TDecompressingConsumerBootstrap<> bootstrap; + bootstrap.Create(); + bootstrap.ExpectIsDeadCall(); + bootstrap.Start(); + + auto isDead = bootstrap.DecompressingConsumer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + bootstrap.DeadPromise.SetValue(TError()); + + isDead.GetValueSync(); // doesn't hang on + } + + Y_UNIT_TEST(PassesNonData) { + TDecompressingConsumerBootstrap<> bootstrap; + bootstrap.Create(); + bootstrap.ExpectIsDeadCall(); + bootstrap.Start(); + + InSequence sequence; + { + TReadResponse resp; + resp.MutableCommit()->AddCookie(42); + + TConsumerMessage ret(std::move(resp)); + + EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) + .WillOnce(Return(ret)); + } + + { + TReadResponse resp; + resp.MutableRelease()->SetTopic("topic!"); + + TConsumerMessage ret(std::move(resp)); + + EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) + .WillOnce(Return(ret)); + } + + auto passed1 = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(passed1.Response.GetCommit().CookieSize(), 1, passed1.Response); + UNIT_ASSERT_VALUES_EQUAL_C(passed1.Response.GetCommit().GetCookie(0), 42, passed1.Response); + + auto passed2 = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync(); + UNIT_ASSERT_STRINGS_EQUAL_C(passed2.Response.GetRelease().GetTopic(), "topic!", passed2.Response); + } + + void ProcessesBrokenChunks(bool skip) { + TDecompressingConsumerBootstrap<> bootstrap; + bootstrap.Settings.SkipBrokenChunks = skip; + bootstrap.Create(); + bootstrap.ExpectIsDeadCall(); + bootstrap.Start(); + + TReadResponse resp; + auto* msg = resp.MutableData()->AddMessageBatch()->AddMessage(); + msg->SetData("hjdhkjhkjhshqsiuhqisuqihsi;"); + msg->MutableMeta()->SetCodec(ECodec::LZOP); + + EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) + .WillOnce(Return(TConsumerMessage(std::move(resp)))); + + auto isDead = bootstrap.DecompressingConsumer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + auto passed = bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync(); + + if (skip) { + UNIT_ASSERT(!passed.Response.HasError()); + isDead.HasValue(); + UNIT_ASSERT_VALUES_EQUAL(passed.Response.GetData().MessageBatchSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(passed.Response.GetData().GetMessageBatch(0).MessageSize(), 1); + UNIT_ASSERT(passed.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData().empty()); + } else { + UNIT_ASSERT(passed.Response.HasError()); + isDead.GetValueSync(); + + UNIT_ASSERT(bootstrap.DecompressingConsumer->GetNextMessage().GetValueSync().Response.HasError()); + } + + //DestroyAndWait(bootstrap.DecompressingConsumer); + } + + Y_UNIT_TEST(DiesOnBrokenChunks) { + ProcessesBrokenChunks(false); + } + + Y_UNIT_TEST(SkipsBrokenChunks) { + ProcessesBrokenChunks(true); + } + + static void AddMessage(TReadResponse::TData::TMessageBatch* batch, const TString& sourceData, ECodec codec = ECodec::RAW) { + auto* msg = batch->AddMessage(); + msg->MutableMeta()->SetCodec(codec); + if (codec == ECodec::RAW) { + msg->SetData(sourceData); + } else { + msg->SetData(TData::Encode(sourceData, codec, -1).GetEncodedData()); + } + } + + Y_UNIT_TEST(DecompessesDataInProperChuncksOrder) { + TDecompressingConsumerBootstrap<> bootstrap; + bootstrap.Create(); + bootstrap.ExpectIsDeadCall(); + bootstrap.Start(); + + InSequence sequence; + { + TReadResponse resp; + auto* batch = resp.MutableData()->AddMessageBatch(); + AddMessage(batch, "message1", ECodec::LZOP); + AddMessage(batch, "message2"); + AddMessage(batch, "message3", ECodec::GZIP); + + resp.MutableData()->AddMessageBatch(); + + batch = resp.MutableData()->AddMessageBatch(); + AddMessage(batch, "messageA", ECodec::LZOP); + AddMessage(batch, "messageB", ECodec::ZSTD); + AddMessage(batch, "messageC"); + + EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) + .WillOnce(Return(TConsumerMessage(std::move(resp)))); + } + + { + TReadResponse resp; + AddMessage(resp.MutableData()->AddMessageBatch(), "trololo", ECodec::LZOP); + + EXPECT_CALL(*bootstrap.MockConsumer, MockGetNextMessage()) + .WillOnce(Return(TConsumerMessage(std::move(resp)))); + } + + auto isDead = bootstrap.DecompressingConsumer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + auto f1 = bootstrap.DecompressingConsumer->GetNextMessage(); + auto f2 = bootstrap.DecompressingConsumer->GetNextMessage(); + + auto data1 = f1.GetValueSync().Response.GetData(); + auto data2 = f2.GetValueSync().Response.GetData(); + + UNIT_ASSERT_VALUES_EQUAL(data1.MessageBatchSize(), 3); + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).MessageSize(), 3); + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(1).MessageSize(), 0); + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).MessageSize(), 3); + + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(0).GetData(), "message1"); + UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW); + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(1).GetData(), "message2"); + UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(1).GetMeta().GetCodec(), ECodec::RAW); + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(0).GetMessage(2).GetData(), "message3"); + UNIT_ASSERT_EQUAL(data1.GetMessageBatch(0).GetMessage(2).GetMeta().GetCodec(), ECodec::RAW); + + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(0).GetData(), "messageA"); + UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW); + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(1).GetData(), "messageB"); + UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(1).GetMeta().GetCodec(), ECodec::RAW); + UNIT_ASSERT_VALUES_EQUAL(data1.GetMessageBatch(2).GetMessage(2).GetData(), "messageC"); + UNIT_ASSERT_EQUAL(data1.GetMessageBatch(2).GetMessage(2).GetMeta().GetCodec(), ECodec::RAW); + + + UNIT_ASSERT_VALUES_EQUAL(data2.MessageBatchSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(data2.GetMessageBatch(0).MessageSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(data2.GetMessageBatch(0).GetMessage(0).GetData(), "trololo"); + UNIT_ASSERT_EQUAL(data2.GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), ECodec::RAW); + + + UNIT_ASSERT(!isDead.HasValue()); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp new file mode 100644 index 0000000000..2ebfcd8d07 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.cpp @@ -0,0 +1,20 @@ +#include "iconsumer_p.h" +#include "persqueue_p.h" + +namespace NPersQueue { + +TPublicConsumer::TPublicConsumer(std::shared_ptr<IConsumerImpl> impl) + : Impl(std::move(impl)) +{ +} + +TPublicConsumer::~TPublicConsumer() { + Impl->Cancel(); +} + +IConsumerImpl::IConsumerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) + : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib)) +{ +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h new file mode 100644 index 0000000000..35c151b9de --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iconsumer_p.h @@ -0,0 +1,51 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h> +#include "interface_common.h" + +namespace NPersQueue { + +class TPQLibPrivate; + +class IConsumerImpl: public IConsumer, public TSyncDestroyed { +public: + IConsumerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); + + virtual void Init() { + } + + using IConsumer::GetNextMessage; + virtual void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept = 0; +}; + +// Consumer that is given to client. +class TPublicConsumer: public IConsumer { +public: + explicit TPublicConsumer(std::shared_ptr<IConsumerImpl> impl); + ~TPublicConsumer(); + + NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override { + return Impl->Start(deadline); + } + + NThreading::TFuture<TConsumerMessage> GetNextMessage() noexcept override { + return Impl->GetNextMessage(); + } + + void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override { + Impl->RequestPartitionStatus(topic, partition, generation); + } + + void Commit(const TVector<ui64>& cookies) noexcept override { + Impl->Commit(cookies); + } + + NThreading::TFuture<TError> IsDead() noexcept override { + return Impl->IsDead(); + } + +private: + std::shared_ptr<IConsumerImpl> Impl; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp new file mode 100644 index 0000000000..e88c5b1280 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.cpp @@ -0,0 +1,31 @@ +#include "interface_common.h" +#include "persqueue_p.h" + +namespace NPersQueue { + +const TString& GetCancelReason() +{ + static const TString reason = "Destroyed"; + return reason; +} + +TSyncDestroyed::TSyncDestroyed(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) + : DestroyEventRef(std::move(destroyEventRef)) + , PQLib(std::move(pqLib)) +{ +} + +TSyncDestroyed::~TSyncDestroyed() { + DestroyedPromise.SetValue(); +} + +void TSyncDestroyed::DestroyPQLibRef() { + auto guard = Guard(DestroyLock); + if (DestroyEventRef) { + IsCanceling = true; + PQLib = nullptr; + DestroyEventRef = nullptr; + } +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h new file mode 100644 index 0000000000..bd3a228b2c --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/interface_common.h @@ -0,0 +1,50 @@ +#pragma once + +#include <library/cpp/threading/future/future.h> + +#include <util/generic/ptr.h> +#include <util/system/spinlock.h> + +#include <memory> + +namespace NPersQueue { + +class TPQLibPrivate; + +const TString& GetCancelReason(); + +class TSyncDestroyed { +protected: + TSyncDestroyed(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); + +public: + // Get future that is signalled after object's destructor. + // Non-pqlib threads can wait on this future to ensure that + // all async operations are finished. + virtual NThreading::TFuture<void> Destroyed() noexcept { + return DestroyedPromise.GetFuture(); + } + + // This destructor will be executed after real object's destructor + // in which it is expected to set all owned promises. + virtual ~TSyncDestroyed(); + + virtual void Cancel() = 0; + + void SetDestroyEventRef(std::shared_ptr<void> ref) { + DestroyEventRef = std::move(ref); + } + +protected: + // Needed for proper PQLib deinitialization + void DestroyPQLibRef(); + +protected: + NThreading::TPromise<void> DestroyedPromise = NThreading::NewPromise<void>(); + std::shared_ptr<void> DestroyEventRef; + TIntrusivePtr<TPQLibPrivate> PQLib; + bool IsCanceling = false; + TAdaptiveLock DestroyLock; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h new file mode 100644 index 0000000000..5c893da75e --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h @@ -0,0 +1,105 @@ +#pragma once +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> + +#include <library/cpp/grpc/common/time_point.h> +#include <library/cpp/threading/future/future.h> + +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/string/cast.h> + +#include <grpc++/channel.h> +#include <grpc++/create_channel.h> + +#include <contrib/libs/grpc/include/grpcpp/impl/codegen/client_context.h> + +#include <chrono> + +#define WRITE_LOG(msg, srcId, sessionId, level, logger) \ + if (logger && logger->IsEnabled(level)) { \ + logger->Log(TStringBuilder() << msg, srcId, sessionId, level); \ + } + +#define DEBUG_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_DEBUG, Logger) +#define INFO_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_INFO, Logger) +#define WARN_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_WARNING, Logger) +#define ERR_LOG(msg, srcId, sessionId) WRITE_LOG(msg, srcId, sessionId, TLOG_ERR, Logger) + +namespace NPersQueue { + +TString GetToken(ICredentialsProvider* credentials); + +void FillMetaHeaders(grpc::ClientContext& context, const TString& database, ICredentialsProvider* credentials); + +bool UseCDS(const TServerSetting& server); + +struct TWriteData { + TProducerSeqNo SeqNo; + TData Data; + + TWriteData(ui64 seqNo, TData&& data) + : SeqNo(seqNo) + , Data(std::move(data)) + {} +}; + +class IQueueEvent { +public: + virtual ~IQueueEvent() = default; + + //! Execute an action defined by implementation. + virtual bool Execute(bool ok) = 0; + + //! Finish and destroy request. + virtual void DestroyRequest() = 0; +}; + +class IHandler : public TAtomicRefCount<IHandler> { +public: + IHandler() + {} + + virtual ~IHandler() + {} + + virtual void Destroy(const TError&) = 0; + virtual void Done() = 0; + virtual TString ToString() = 0; +}; + +using IHandlerPtr = TIntrusivePtr<IHandler>; + +class TQueueEvent : public IQueueEvent { +public: + TQueueEvent(IHandlerPtr handler) + : Handler(std::move(handler)) + { + Y_ASSERT(Handler); + } + +private: + bool Execute(bool ok) override { + if (ok) { + Handler->Done(); + } else { + TError error; + error.SetDescription("event " + Handler->ToString() + " failed"); + error.SetCode(NErrorCode::ERROR); + Handler->Destroy(error); + } + return false; + } + + virtual ~TQueueEvent() { + Handler.Reset(); + } + + void DestroyRequest() override + { + delete this; + } + + IHandlerPtr Handler; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp new file mode 100644 index 0000000000..f245df74fe --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.cpp @@ -0,0 +1,20 @@ +#include "iprocessor_p.h" +#include "persqueue_p.h" + +namespace NPersQueue { + +TPublicProcessor::TPublicProcessor(std::shared_ptr<IProcessorImpl> impl) + : Impl(std::move(impl)) +{ +} + +TPublicProcessor::~TPublicProcessor() { + Impl->Cancel(); +} + +IProcessorImpl::IProcessorImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) + : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib)) +{ +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h new file mode 100644 index 0000000000..e315b8412f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iprocessor_p.h @@ -0,0 +1,35 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> +#include "interface_common.h" + +namespace NPersQueue { + +class TPQLibPrivate; + +struct IProcessorImpl: public IProcessor, public TSyncDestroyed { + using IProcessor::GetNextData; + + IProcessorImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); + + // Initialization after constructor (for example, for correct call of shared_from_this()) + virtual void Init() { + } + virtual void GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept = 0; +}; + +// Processor that is given to client. +class TPublicProcessor: public IProcessor { +public: + explicit TPublicProcessor(std::shared_ptr<IProcessorImpl> impl); + ~TPublicProcessor(); + + NThreading::TFuture<TOriginData> GetNextData() noexcept override { + return Impl->GetNextData(); + } + +private: + std::shared_ptr<IProcessorImpl> Impl; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp new file mode 100644 index 0000000000..c7a267b862 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.cpp @@ -0,0 +1,20 @@ +#include "iproducer_p.h" +#include "persqueue_p.h" + +namespace NPersQueue { + +TPublicProducer::TPublicProducer(std::shared_ptr<IProducerImpl> impl) + : Impl(std::move(impl)) +{ +} + +TPublicProducer::~TPublicProducer() { + Impl->Cancel(); +} + +IProducerImpl::IProducerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib) + : TSyncDestroyed(std::move(destroyEventRef), std::move(pqLib)) +{ +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h new file mode 100644 index 0000000000..437798c551 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h @@ -0,0 +1,58 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> +#include "interface_common.h" + +namespace NPersQueue { + +class TPQLibPrivate; + +struct IProducerImpl: public IProducer, public TSyncDestroyed { + using IProducer::Write; + + IProducerImpl(std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib); + + // Initialization after constructor (for example, for correct call of shared_from_this()) + virtual void Init() { + } + virtual void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept = 0; + virtual void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept = 0; + + NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo, TData) noexcept override { + Y_FAIL(""); + return NThreading::NewPromise<TProducerCommitResponse>().GetFuture(); + } + + NThreading::TFuture<TProducerCommitResponse> Write(TData) noexcept override { + Y_FAIL(""); + return NThreading::NewPromise<TProducerCommitResponse>().GetFuture(); + } +}; + +// Producer that is given to client. +class TPublicProducer: public IProducer { +public: + explicit TPublicProducer(std::shared_ptr<IProducerImpl> impl); + ~TPublicProducer(); + + NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept { + return Impl->Start(deadline); + } + + NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept { + return Impl->Write(seqNo, std::move(data)); + } + + NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept { + return Impl->Write(std::move(data)); + } + + NThreading::TFuture<TError> IsDead() noexcept { + return Impl->IsDead(); + } + +private: + std::shared_ptr<IProducerImpl> Impl; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h new file mode 100644 index 0000000000..2ac0b3ec18 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/local_caller.h @@ -0,0 +1,303 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include "interface_common.h" +#include "internals.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include "persqueue_p.h" + +namespace NPersQueue { + +template <class TImpl> +class TCancelCaller: public TImpl { +public: + using TImpl::TImpl; + + std::shared_ptr<TCancelCaller> MakeSharedRef() { + return std::static_pointer_cast<TCancelCaller>(this->shared_from_this()); + } + + void Cancel() override { + auto guard = Guard(this->DestroyLock); + if (!this->IsCanceling) { + this->IsCanceling = true; + AddCancelCaller(); + } + } + +protected: + const TImpl* QueueTag() const { // we must have exactly the same thread tag as TImpl object + return this; + } + + static void MakeCanceledError(TError& error) { + error.SetDescription(GetCancelReason()); + error.SetCode(NErrorCode::ERROR); + } + + template <class TResponseProto> + static void MakeCanceledError(TResponseProto& resp) { + MakeCanceledError(*resp.MutableError()); + } + + template <class TResponse, class... TArgs> + static void MakeCanceledError(NThreading::TPromise<TResponse>& promise, TArgs&&...) { // the most common case for calls + decltype(promise.GetValue().Response) responseProto; + MakeCanceledError(responseProto); + promise.SetValue(TResponse(std::move(responseProto))); + } + + template <class TResponse> + static TResponse MakeCanceledError() { + TResponse resp; + MakeCanceledError(*resp.MutableError()); + return resp; + } + +private: + void AddCancelCaller() { + std::weak_ptr<TCancelCaller> selfWeak = MakeSharedRef(); + auto caller = [selfWeak]() mutable { + auto selfShared = selfWeak.lock(); + if (selfShared) { + selfShared->TImpl::Cancel(); + } + }; + Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(QueueTag()).AddFunc(caller)); + } +}; + +template <class TImpl, class TStartResponse> +class TLocalStartDeadImplCaller: public TCancelCaller<TImpl> { +protected: + using TCancelCaller<TImpl>::TCancelCaller; + + NThreading::TFuture<TStartResponse> Start(TInstant deadline) noexcept override { + IsDead(); // subscribe for impl future + auto guard = Guard(this->DestroyLock); + if (this->IsCanceling) { + guard.Release(); + decltype(reinterpret_cast<TStartResponse*>(0)->Response) respProto; + this->MakeCanceledError(respProto); + return NThreading::MakeFuture(TStartResponse(std::move(respProto))); + } + if (StartPromise.Initialized()) { + guard.Release(); + decltype(reinterpret_cast<TStartResponse*>(0)->Response) respProto; + TError& error = *respProto.MutableError(); + error.SetDescription("Start was already called"); + error.SetCode(NErrorCode::BAD_REQUEST); + return NThreading::MakeFuture(TStartResponse(std::move(respProto))); + } + StartPromise = NThreading::NewPromise<TStartResponse>(); + AddStartCaller(deadline); + return StartPromise.GetFuture(); + } + + NThreading::TFuture<TError> IsDead() noexcept override { + auto guard = Guard(this->DestroyLock); + if (this->IsCanceling && !IsDeadCalled) { + guard.Release(); + TError err; + this->MakeCanceledError(err); + return NThreading::MakeFuture(err); + } + if (!IsDeadCalled) { + AddIsDeadCaller(); + } + return DeadPromise.GetFuture(); + } + +private: + std::shared_ptr<TLocalStartDeadImplCaller> MakeSharedRef() { + return std::static_pointer_cast<TLocalStartDeadImplCaller>(this->shared_from_this()); + } + + void AddIsDeadCaller() { + IsDeadCalled = true; + std::shared_ptr<TLocalStartDeadImplCaller> selfShared = MakeSharedRef(); + auto isDeadCaller = [selfShared]() mutable { + selfShared->CallIsDead(); + }; + Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(isDeadCaller)); + } + + void CallIsDead() { + auto deadPromise = DeadPromise; + TImpl::IsDead().Subscribe([deadPromise](const auto& future) mutable { + deadPromise.SetValue(future.GetValue()); + }); + } + + void AddStartCaller(TInstant deadline) { + std::shared_ptr<TLocalStartDeadImplCaller> selfShared = MakeSharedRef(); + auto startCaller = [selfShared, deadline]() mutable { + selfShared->CallStart(deadline); + }; + Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(startCaller)); + } + + void CallStart(TInstant deadline) { + auto startPromise = StartPromise; + TImpl::Start(deadline).Subscribe([startPromise](const auto& future) mutable { + startPromise.SetValue(future.GetValue()); + }); + } + +private: + NThreading::TPromise<TStartResponse> StartPromise; + NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); + bool IsDeadCalled = false; +}; + +#define LOCAL_CALLER(Func, Args, DeclTail, ...) \ + void Func Args DeclTail { \ + auto guard = Guard(this->DestroyLock); \ + if (this->IsCanceling) { \ + guard.Release(); \ + this->MakeCanceledError(__VA_ARGS__); \ + return; \ + } \ + auto selfShared = MakeSharedRef(); \ + auto caller = [selfShared, __VA_ARGS__]() mutable { \ + selfShared->TImpl::Func(__VA_ARGS__); \ + }; \ + Y_VERIFY(this->PQLib->GetQueuePool().GetQueue(this->QueueTag()).AddFunc(caller));\ + } + +// Assumes that there is function with promise interface +// first arg must be promise +#define LOCAL_CALLER_WITH_FUTURE(ReturnType, Func, Args, DeclTail, ...) \ + NThreading::TFuture<ReturnType> Func Args DeclTail { \ + auto promise = NThreading::NewPromise<ReturnType>(); \ + Func(__VA_ARGS__); \ + return promise.GetFuture(); \ + } + +// Helper classes for producer/consumer/processor impls. +// Wrappers that delegate calls to its methods to proper thread pool local threads. +// The result is that all calls to impl are serialized and locks are no more needed. +// +// Implies: +// That impl class is std::enable_shared_from_this descendant. + +template <class TImpl> +class TLocalProducerImplCaller: public TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse> { +public: + using TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse>::TLocalStartDeadImplCaller; + + LOCAL_CALLER(Write, + (NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data), + noexcept override, + promise, seqNo, data) + + LOCAL_CALLER(Write, + (NThreading::TPromise<TProducerCommitResponse>& promise, TData data), + noexcept override, + promise, data) + + LOCAL_CALLER_WITH_FUTURE(TProducerCommitResponse, + Write, + (TProducerSeqNo seqNo, TData data), + noexcept override, + promise, seqNo, data) + + LOCAL_CALLER_WITH_FUTURE(TProducerCommitResponse, + Write, + (TData data), + noexcept override, + promise, data) + +protected: + using TLocalStartDeadImplCaller<TImpl, TProducerCreateResponse>::MakeCanceledError; + static void MakeCanceledError(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) { + TWriteResponse responseProto; + MakeCanceledError(responseProto); + promise.SetValue(TProducerCommitResponse(seqNo, std::move(data), std::move(responseProto))); + } + + static void MakeCanceledError(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) { + MakeCanceledError(promise, TProducerSeqNo(0), std::move(data)); + } + +private: + std::shared_ptr<TLocalProducerImplCaller> MakeSharedRef() { + return std::static_pointer_cast<TLocalProducerImplCaller>(this->shared_from_this()); + } +}; + +template <class TImpl> +class TLocalConsumerImplCaller: public TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse> { +public: + using TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse>::TLocalStartDeadImplCaller; + + LOCAL_CALLER(GetNextMessage, + (NThreading::TPromise<TConsumerMessage>& promise), + noexcept override, + promise) + + LOCAL_CALLER_WITH_FUTURE(TConsumerMessage, + GetNextMessage, + (), + noexcept override, + promise) + + LOCAL_CALLER(Commit, + (const TVector<ui64>& cookies), + noexcept override, + cookies) + + LOCAL_CALLER(RequestPartitionStatus, + (const TString& topic, ui64 partition, ui64 generation), + noexcept override, + topic, partition, generation) + +private: + using TLocalStartDeadImplCaller<TImpl, TConsumerCreateResponse>::MakeCanceledError; + static void MakeCanceledError(const TVector<ui64>& cookies) { + Y_UNUSED(cookies); + // Do nothing, because this method doesn't return future + } + + static void MakeCanceledError(const TString&, ui64, ui64) { + // Do nothing, because this method doesn't return future + } + + + std::shared_ptr<TLocalConsumerImplCaller> MakeSharedRef() { + return std::static_pointer_cast<TLocalConsumerImplCaller>(this->shared_from_this()); + } +}; + +template <class TImpl> +class TLocalProcessorImplCaller: public TCancelCaller<TImpl> { +public: + using TCancelCaller<TImpl>::TCancelCaller; + + LOCAL_CALLER(GetNextData, + (NThreading::TPromise<TOriginData>& promise), + noexcept override, + promise) + + LOCAL_CALLER_WITH_FUTURE(TOriginData, + GetNextData, + (), + noexcept override, + promise) + +private: + using TCancelCaller<TImpl>::MakeCanceledError; + static void MakeCanceledError(NThreading::TPromise<TOriginData>& promise) { + promise.SetValue(TOriginData()); // with empty data + } + + std::shared_ptr<TLocalProcessorImplCaller> MakeSharedRef() { + return std::static_pointer_cast<TLocalProcessorImplCaller>(this->shared_from_this()); + } +}; + +#undef LOCAL_CALLER +#undef LOCAL_CALLER_WITH_FUTURE + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp new file mode 100644 index 0000000000..98d44d14c5 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/logger.cpp @@ -0,0 +1,40 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h> +#include <util/datetime/base.h> +#include <util/string/builder.h> +#include <util/stream/str.h> + +namespace NPersQueue { + +static const TStringBuf LogLevelsStrings[] = { + "EMERG", + "ALERT", + "CRITICAL_INFO", + "ERROR", + "WARNING", + "NOTICE", + "INFO", + "DEBUG", +}; + +TStringBuf TCerrLogger::LevelToString(int level) { + return LogLevelsStrings[ClampVal(level, 0, int(Y_ARRAY_SIZE(LogLevelsStrings) - 1))]; +} + +void TCerrLogger::Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) { + if (level > Level) { + return; + } + + TStringBuilder message; + message << TInstant::Now() << " :" << LevelToString(level) << ":"; + if (sourceId) { + message << " SourceId [" << sourceId << "]:"; + } + if (sessionId) { + message << " SessionId [" << sessionId << "]:"; + } + message << " " << msg << "\n"; + Cerr << message; +} + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp new file mode 100644 index 0000000000..cd986532dc --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.cpp @@ -0,0 +1,645 @@ +#include "multicluster_consumer.h" +#include "persqueue_p.h" + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +#include <library/cpp/containers/intrusive_rb_tree/rb_tree.h> + +#include <util/generic/guid.h> +#include <util/generic/strbuf.h> + +namespace NPersQueue { + +struct TMultiClusterConsumer::TSubconsumerInfo { + TSubconsumerInfo(std::shared_ptr<IConsumerImpl> consumer); + + std::shared_ptr<IConsumerImpl> Consumer; + NThreading::TFuture<TConsumerCreateResponse> StartFuture; + NThreading::TFuture<TError> DeadFuture; + bool StartResponseWasGot = false; // We call methods of this consumer only after this flag was set. + + std::deque<NThreading::TFuture<TConsumerMessage>> PendingRequests; +}; + +namespace { + +struct TConsumerCookieKey; +struct TUserCookieMappingItem; +struct TConsumerCookieMappingItem; + +struct TConsumerCookieKey { + size_t Subconsumer = 0; + ui64 ConsumerCookie = 0; + + TConsumerCookieKey(size_t subconsumer, ui64 consumerCookie) + : Subconsumer(subconsumer) + , ConsumerCookie(consumerCookie) + { + } + + bool operator<(const TConsumerCookieKey& other) const { + return std::make_pair(Subconsumer, ConsumerCookie) < std::make_pair(other.Subconsumer, other.ConsumerCookie); + } +}; + +struct TUserCookieCmp { + static ui64 GetKey(const TUserCookieMappingItem& item); + + static bool Compare(ui64 l, const TUserCookieMappingItem& r) { + return l < GetKey(r); + } + + static bool Compare(const TUserCookieMappingItem& l, ui64 r) { + return GetKey(l) < r; + } + + static bool Compare(const TUserCookieMappingItem& l, const TUserCookieMappingItem& r) { + return GetKey(l) < GetKey(r); + } +}; + +struct TConsumerCookieCmp { + static TConsumerCookieKey GetKey(const TConsumerCookieMappingItem& item); + + static bool Compare(const TConsumerCookieKey& l, const TConsumerCookieMappingItem& r) { + return l < GetKey(r); + } + + static bool Compare(const TConsumerCookieMappingItem& l, const TConsumerCookieKey& r) { + return GetKey(l) < r; + } + + static bool Compare(const TConsumerCookieMappingItem& l, const TConsumerCookieMappingItem& r) { + return GetKey(l) < GetKey(r); + } +}; + +struct TUserCookieMappingItem : public TRbTreeItem<TUserCookieMappingItem, TUserCookieCmp> { +}; + +struct TConsumerCookieMappingItem : public TRbTreeItem<TConsumerCookieMappingItem, TConsumerCookieCmp> { +}; + +bool IsRetryable(const grpc::Status& status) { + switch (status.error_code()) { + case grpc::OK: + case grpc::CANCELLED: + case grpc::INVALID_ARGUMENT: + case grpc::NOT_FOUND: + case grpc::ALREADY_EXISTS: + case grpc::PERMISSION_DENIED: + case grpc::UNAUTHENTICATED: + case grpc::FAILED_PRECONDITION: + case grpc::ABORTED: + case grpc::OUT_OF_RANGE: + case grpc::UNIMPLEMENTED: + return false; + + case grpc::UNKNOWN: + case grpc::DEADLINE_EXCEEDED: + case grpc::RESOURCE_EXHAUSTED: + case grpc::INTERNAL: + case grpc::UNAVAILABLE: + case grpc::DATA_LOSS: + case grpc::DO_NOT_USE: + return true; + } +} + +} // namespace + +struct TMultiClusterConsumer::TCookieMappingItem : public TUserCookieMappingItem, public TConsumerCookieMappingItem { + TCookieMappingItem(size_t subconsumer, ui64 consumerCookie, ui64 userCookie) + : Subconsumer(subconsumer) + , ConsumerCookie(consumerCookie) + , UserCookie(userCookie) + { + } + + size_t Subconsumer = 0; + ui64 ConsumerCookie = 0; + ui64 UserCookie = 0; +}; + +namespace { + +ui64 TUserCookieCmp::GetKey(const TUserCookieMappingItem& item) { + return static_cast<const TMultiClusterConsumer::TCookieMappingItem&>(item).UserCookie; +} + +TConsumerCookieKey TConsumerCookieCmp::GetKey(const TConsumerCookieMappingItem& item) { + const auto& src = static_cast<const TMultiClusterConsumer::TCookieMappingItem&>(item); + return { src.Subconsumer, src.ConsumerCookie }; +} + +} // namespace + +struct TMultiClusterConsumer::TCookieMapping { + using TUserCookieTree = TRbTree<TUserCookieMappingItem, TUserCookieCmp>; + using TConsumerCookieTree = TRbTree<TConsumerCookieMappingItem, TConsumerCookieCmp>; + + struct TConsumerCookieDestroy : public TConsumerCookieTree::TDestroy { + void operator()(TConsumerCookieMappingItem& item) const { + TDestroy::operator()(item); // Remove from tree + delete static_cast<TCookieMappingItem*>(&item); + } + }; + + ~TCookieMapping() { + UserCookieTree.ForEachNoOrder(TUserCookieTree::TDestroy()); + UserCookieTree.Init(); + + ConsumerCookieTree.ForEachNoOrder(TConsumerCookieDestroy()); + ConsumerCookieTree.Init(); + } + + ui64 AddMapping(size_t subconsumer, ui64 cookie) { // Returns user cookie + auto* newItem = new TCookieMappingItem(subconsumer, cookie, NextCookie++); + UserCookieTree.Insert(newItem); + ConsumerCookieTree.Insert(newItem); + return newItem->UserCookie; + } + + TCookieMappingItem* FindMapping(ui64 userCookie) { + return DownCast(UserCookieTree.Find(userCookie)); + } + + TCookieMappingItem* FindMapping(size_t subconsumer, ui64 cookie) { + return DownCast(ConsumerCookieTree.Find(TConsumerCookieKey(subconsumer, cookie))); + } + + void RemoveUserCookieMapping(TCookieMappingItem* item) { + Y_ASSERT(item); + static_cast<TUserCookieMappingItem*>(item)->UnLink(); + } + + void RemoveMapping(TCookieMappingItem* item) { + Y_ASSERT(item); + Y_ASSERT(!static_cast<TUserCookieMappingItem*>(item)->ParentTree()); + static_cast<TUserCookieMappingItem*>(item)->UnLink(); // Just in case + static_cast<TConsumerCookieMappingItem*>(item)->UnLink(); + delete item; + } + +private: + template <class T> + static TCookieMappingItem* DownCast(T* item) { + if (item) { + return static_cast<TCookieMappingItem*>(item); + } else { + return nullptr; + } + } + + TUserCookieTree UserCookieTree; + TConsumerCookieTree ConsumerCookieTree; + ui64 NextCookie = 1; +}; + +TMultiClusterConsumer::TMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) + : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) + , Settings(settings) + , Logger(std::move(logger)) + , SessionId(CreateGuidAsString()) + , CookieMapping(MakeHolder<TCookieMapping>()) +{ + PatchSettings(); +} + +TMultiClusterConsumer::~TMultiClusterConsumer() { + Destroy(); +} + +void TMultiClusterConsumer::PatchSettings() { + if (!Settings.ReconnectOnFailure) { + WARN_LOG("Ignoring ReconnectOnFailure=false option for multicluster consumer", "", SessionId); + Settings.ReconnectOnFailure = true; + } + if (Settings.MaxAttempts != std::numeric_limits<unsigned>::max()) { + WARN_LOG("Ignoring MaxAttempts option for multicluster consumer", "", SessionId); + Settings.MaxAttempts = std::numeric_limits<unsigned>::max(); + } +} + +NThreading::TFuture<TConsumerCreateResponse> TMultiClusterConsumer::Start(TInstant deadline) noexcept { + Y_VERIFY(State == EState::Created); + StartClusterDiscovery(deadline); + return StartPromise.GetFuture(); +} + +NThreading::TFuture<TError> TMultiClusterConsumer::IsDead() noexcept { + return IsDeadPromise.GetFuture(); +} + +void TMultiClusterConsumer::ScheduleClusterDiscoveryRetry(TInstant deadline) { + const TInstant now = TInstant::Now(); + if (ClusterDiscoveryAttemptsDone >= Settings.MaxAttempts || now >= deadline) { + Destroy(TStringBuilder() << "Failed " << ClusterDiscoveryAttemptsDone << " cluster discovery attempts", NErrorCode::CREATE_TIMEOUT); + return; + } + + if (!IsRetryable(ClusterDiscoverResult->first)) { + Destroy(TStringBuilder() << "Cluster discovery failed: " << static_cast<int>(ClusterDiscoverResult->first.error_code())); + return; + } + + TDuration delay = Min(Settings.MaxReconnectionDelay, ClusterDiscoveryAttemptsDone * Settings.ReconnectionDelay, deadline - now); + std::weak_ptr<TMultiClusterConsumer> weakRef = shared_from_this(); + PQLib->GetScheduler().Schedule(delay, this, [weakRef, deadline]() { + if (auto self = weakRef.lock()) { + self->StartClusterDiscovery(deadline); + } + }); +} + +void TMultiClusterConsumer::StartClusterDiscovery(TInstant deadline) { + if (State == EState::Dead) { + return; + } + + State = EState::WaitingClusterDiscovery; + ++ClusterDiscoveryAttemptsDone; + DEBUG_LOG("Starting cluster discovery", "", SessionId); + + auto discoverer = MakeIntrusive<TConsumerChannelOverCdsImpl>(Settings, PQLib.Get(), Logger); + discoverer->Start(); + ClusterDiscoverResult = discoverer->GetResultPtr(); + ClusterDiscoverer = std::move(discoverer); + + std::weak_ptr<TMultiClusterConsumer> self = shared_from_this(); + auto handler = [self, deadline](const auto&) { + if (auto selfShared = self.lock()) { + selfShared->OnClusterDiscoveryDone(deadline); + } + }; + PQLib->Subscribe(ClusterDiscoverer->GetChannel(), this, handler); +} + +void TMultiClusterConsumer::OnClusterDiscoveryDone(TInstant deadline) { + if (State != EState::WaitingClusterDiscovery) { + return; + } + + ClusterDiscoverer = nullptr; // delete + if (!ClusterDiscoverResult->first.ok()) { + INFO_LOG("Failed to discover clusters. Grpc error: " << static_cast<int>(ClusterDiscoverResult->first.error_code()), "", SessionId); + ScheduleClusterDiscoveryRetry(deadline); // Destroys if we shouldn't retry. + return; + } + + if (static_cast<size_t>(ClusterDiscoverResult->second.read_sessions_clusters_size()) != Settings.Topics.size()) { + Destroy("Got unexpected cluster discovery result"); + return; + } + + if (deadline != TInstant::Max()) { + const TInstant now = TInstant::Now(); + if (now > deadline) { + Destroy("Start timeout", NErrorCode::CREATE_TIMEOUT); + return; + } + + std::weak_ptr<TMultiClusterConsumer> self = shared_from_this(); + auto onStartTimeout = [self] { + if (auto selfShared = self.lock()) { + selfShared->OnStartTimeout(); + } + }; + StartDeadlineCallback = + PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout); + } + + State = EState::StartingSubconsumers; + DEBUG_LOG("Starting subconsumers", "", SessionId); + + // Group topics by clusters. + THashMap<TString, TVector<TString>> clusterToTopics; + for (size_t topicIndex = 0, topicsCount = Settings.Topics.size(); topicIndex < topicsCount; ++topicIndex) { + const Ydb::PersQueue::ClusterDiscovery::ReadSessionClusters& clusters = ClusterDiscoverResult->second.read_sessions_clusters(topicIndex); + const TString& topic = Settings.Topics[topicIndex]; + for (const Ydb::PersQueue::ClusterDiscovery::ClusterInfo& clusterInfo : clusters.clusters()) { + clusterToTopics[clusterInfo.endpoint()].push_back(topic); + } + } + + if (clusterToTopics.empty()) { + Destroy("Got empty endpoint set from cluster discovery"); + return; + } + + // Start consumer on every cluster with its topics set. + for (auto&& [clusterEndpoint, topics] : clusterToTopics) { + // Settings. + TConsumerSettings settings = Settings; + settings.Server = ApplyClusterEndpoint(settings.Server, clusterEndpoint); + settings.ReadFromAllClusterSources = false; + settings.Topics = std::move(topics); + settings.Unpack = false; // Unpack is being done in upper level decompressing consumer. + settings.MaxMemoryUsage = Max(settings.MaxMemoryUsage / clusterToTopics.size(), static_cast<size_t>(1)); + if (settings.MaxUncommittedSize > 0) { // Limit is enabled. + settings.MaxUncommittedSize = Max(settings.MaxUncommittedSize / clusterToTopics.size(), static_cast<size_t>(1)); + } + if (settings.MaxUncommittedCount > 0) { // Limit is enabled. + settings.MaxUncommittedCount = Max(settings.MaxUncommittedCount / clusterToTopics.size(), static_cast<size_t>(1)); + } + const size_t subconsumerIndex = Subconsumers.size(); + + // Create subconsumer. + Subconsumers.push_back(PQLib->CreateRawRetryingConsumer(settings, DestroyEventRef, Logger)); + + // Subscribe on start. + std::weak_ptr<TMultiClusterConsumer> self = shared_from_this(); + auto handler = [self, subconsumerIndex](const auto&) { + if (auto selfShared = self.lock()) { + selfShared->OnSubconsumerStarted(subconsumerIndex); + } + }; + PQLib->Subscribe(Subconsumers.back().StartFuture, this, handler); + } +} + +void TMultiClusterConsumer::OnStartTimeout() { + if (State == EState::Dead || State == EState::Working) { + return; + } + + StartDeadlineCallback = nullptr; + Destroy("Start timeout", NErrorCode::CREATE_TIMEOUT); +} + + +void TMultiClusterConsumer::OnSubconsumerStarted(size_t subconsumerIndex) { + if (State == EState::Dead) { + return; + } + + auto& subconsumerInfo = Subconsumers[subconsumerIndex]; + const auto& result = subconsumerInfo.StartFuture.GetValue(); + if (result.Response.HasError()) { + WARN_LOG("Got error on starting subconsumer: " << result.Response.GetError(), "", SessionId); + Destroy(result.Response.GetError()); + return; + } + + // Process start response + subconsumerInfo.StartResponseWasGot = true; + INFO_LOG("Subconsumer " << subconsumerIndex << " started session with id " << result.Response.GetInit().GetSessionId(), "", SessionId); + + // Move to new state + if (State == EState::StartingSubconsumers) { + State = EState::Working; + if (StartDeadlineCallback) { + StartDeadlineCallback->TryCancel(); + StartDeadlineCallback = nullptr; + } + + { + TReadResponse resp; + resp.MutableInit()->SetSessionId(SessionId); + StartPromise.SetValue(TConsumerCreateResponse(std::move(resp))); + } + } else { + RequestSubconsumers(); // Make requests for new subconsumer. + } +} + +void TMultiClusterConsumer::Commit(const TVector<ui64>& cookies) noexcept { + if (State != EState::Working) { + Destroy("Requesting commit, but consumer is not in working state", NErrorCode::BAD_REQUEST); + return; + } + + TVector<TVector<ui64>> subconsumersCookies(Subconsumers.size()); + for (ui64 userCookie : cookies) { + TCookieMappingItem* mapping = CookieMapping->FindMapping(userCookie); + if (!mapping) { + Destroy(TStringBuilder() << "Wrong cookie " << userCookie, NErrorCode::WRONG_COOKIE); + return; + } + Y_ASSERT(mapping->Subconsumer < Subconsumers.size()); + subconsumersCookies[mapping->Subconsumer].push_back(mapping->ConsumerCookie); + CookieMapping->RemoveUserCookieMapping(mapping); // Avoid double commit. This will ensure error on second commit with the same cookie. + } + + for (size_t subconsumerIndex = 0; subconsumerIndex < Subconsumers.size(); ++subconsumerIndex) { + const TVector<ui64>& subconsumerCommitRequest = subconsumersCookies[subconsumerIndex]; + if (!subconsumerCommitRequest.empty()) { + Subconsumers[subconsumerIndex].Consumer->Commit(subconsumerCommitRequest); + } + } +} + +void TMultiClusterConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { + if (State != EState::Working) { + promise.SetValue(MakeResponse(MakeError("Requesting next message, but consumer is not in working state", NErrorCode::BAD_REQUEST))); + return; + } + + Requests.push_back(promise); + CheckReadyResponses(); + RequestSubconsumers(); +} + +void TMultiClusterConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { + if (State != EState::Working) { + Destroy("Requesting partition status, but consumer is not in working state", NErrorCode::BAD_REQUEST); + return; + } + + const auto subconsumerIdIt = OldTopicName2Subconsumer.find(topic); + if (subconsumerIdIt != OldTopicName2Subconsumer.end()) { + Y_ASSERT(subconsumerIdIt->second < Subconsumers.size()); + Subconsumers[subconsumerIdIt->second].Consumer->RequestPartitionStatus(topic, partition, generation); + } else { + WARN_LOG("Requested partition status for topic \"" << topic << "\" (partition " << partition << ", generation " << generation << "), but there is no such lock session. Ignoring request", "", SessionId); + } +} + +void TMultiClusterConsumer::CheckReadyResponses() { + const size_t prevCurrentSubconsumer = CurrentSubconsumer; + + do { + if (!ProcessReadyResponses(CurrentSubconsumer)) { + break; + } + + // Next subconsumer in round robin way. + ++CurrentSubconsumer; + if (CurrentSubconsumer == Subconsumers.size()) { + CurrentSubconsumer = 0; + } + } while (CurrentSubconsumer != prevCurrentSubconsumer && !Requests.empty()); +} + +bool TMultiClusterConsumer::ProcessReadyResponses(size_t subconsumerIndex) { + if (Subconsumers.empty()) { + Y_VERIFY(State == EState::Dead); + return false; + } + Y_VERIFY(subconsumerIndex < Subconsumers.size()); + TSubconsumerInfo& consumerInfo = Subconsumers[subconsumerIndex]; + while (!consumerInfo.PendingRequests.empty() && !Requests.empty() && consumerInfo.PendingRequests.front().HasValue()) { + if (!TranslateConsumerMessage(Requests.front(), consumerInfo.PendingRequests.front().ExtractValue(), subconsumerIndex)) { + return false; + } + Requests.pop_front(); + consumerInfo.PendingRequests.pop_front(); + } + return true; +} + +bool TMultiClusterConsumer::TranslateConsumerMessage(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { + switch (subconsumerResponse.Type) { + case EMT_LOCK: + return TranslateConsumerMessageLock(promise, std::move(subconsumerResponse), subconsumerIndex); + case EMT_RELEASE: + return TranslateConsumerMessageRelease(promise, std::move(subconsumerResponse), subconsumerIndex); + case EMT_DATA: + return TranslateConsumerMessageData(promise, std::move(subconsumerResponse), subconsumerIndex); + case EMT_ERROR: + return TranslateConsumerMessageError(promise, std::move(subconsumerResponse), subconsumerIndex); + case EMT_STATUS: + return TranslateConsumerMessageStatus(promise, std::move(subconsumerResponse), subconsumerIndex); + case EMT_COMMIT: + return TranslateConsumerMessageCommit(promise, std::move(subconsumerResponse), subconsumerIndex); + } +} + +bool TMultiClusterConsumer::TranslateConsumerMessageData(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { + auto* data = subconsumerResponse.Response.MutableData(); + data->SetCookie(CookieMapping->AddMapping(subconsumerIndex, data->GetCookie())); + promise.SetValue(std::move(subconsumerResponse)); + return true; +} + +bool TMultiClusterConsumer::TranslateConsumerMessageCommit(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { + for (ui64& consumerCookie : *subconsumerResponse.Response.MutableCommit()->MutableCookie()) { + auto* mapping = CookieMapping->FindMapping(subconsumerIndex, consumerCookie); + if (!mapping) { + Destroy(TStringBuilder() << "Received unknown cookie " << consumerCookie << " commit"); + return false; + } + Y_VERIFY(mapping); + consumerCookie = mapping->UserCookie; + CookieMapping->RemoveMapping(mapping); // Invalidates mapping + } + promise.SetValue(std::move(subconsumerResponse)); + return true; +} + +bool TMultiClusterConsumer::TranslateConsumerMessageLock(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { + OldTopicName2Subconsumer[subconsumerResponse.Response.GetLock().GetTopic()] = subconsumerIndex; + promise.SetValue(std::move(subconsumerResponse)); + return true; +} + +bool TMultiClusterConsumer::TranslateConsumerMessageRelease(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { + Y_UNUSED(subconsumerIndex); + promise.SetValue(std::move(subconsumerResponse)); + return true; +} + +bool TMultiClusterConsumer::TranslateConsumerMessageError(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { + WARN_LOG("Got error from subconsumer " << subconsumerIndex << ": " << subconsumerResponse.Response.GetError(), "", SessionId); + Destroy(subconsumerResponse.Response.GetError()); + Y_UNUSED(promise); + return false; +} + +bool TMultiClusterConsumer::TranslateConsumerMessageStatus(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex) { + Y_UNUSED(subconsumerIndex); + promise.SetValue(std::move(subconsumerResponse)); + return true; +} + +void TMultiClusterConsumer::RequestSubconsumers() { + const size_t inflight = Requests.size(); + if (!inflight) { + return; + } + TMaybe<std::weak_ptr<TMultiClusterConsumer>> maybeSelf; + for (size_t subconsumerIndex = 0; subconsumerIndex < Subconsumers.size(); ++subconsumerIndex) { + TSubconsumerInfo& consumerInfo = Subconsumers[subconsumerIndex]; + if (!consumerInfo.StartResponseWasGot) { + continue; // Consumer hasn't started yet. + } + while (consumerInfo.PendingRequests.size() < inflight) { + if (!maybeSelf) { + maybeSelf.ConstructInPlace(shared_from_this()); // Don't construct it if we don't need it. + } + consumerInfo.PendingRequests.push_back(consumerInfo.Consumer->GetNextMessage()); + PQLib->Subscribe(consumerInfo.PendingRequests.back(), + this, + [self = *maybeSelf, subconsumerIndex](const auto&) { + if (auto selfShared = self.lock()) { + selfShared->ProcessReadyResponses(subconsumerIndex); + } + }); + } + } +} + +void TMultiClusterConsumer::Destroy(const TError& description) { + if (State == EState::Dead) { + return; + } + State = EState::Dead; + + WARN_LOG("Destroying consumer with error description: " << description, "", SessionId); + + StartPromise.TrySetValue(MakeCreateResponse(description)); + + if (StartDeadlineCallback) { + StartDeadlineCallback->TryCancel(); + } + + for (auto& reqPromise : Requests) { + reqPromise.SetValue(MakeResponse(description)); + } + Requests.clear(); + Subconsumers.clear(); + + IsDeadPromise.SetValue(description); + + DestroyPQLibRef(); +} + +void TMultiClusterConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) { + Destroy(MakeError(description, code)); +} + +void TMultiClusterConsumer::Destroy() { + Destroy(GetCancelReason()); +} + +void TMultiClusterConsumer::Cancel() { + Destroy(GetCancelReason()); +} + +TError TMultiClusterConsumer::MakeError(const TString& description, NErrorCode::EErrorCode code) { + TError error; + error.SetDescription(description); + error.SetCode(code); + return error; +} + +TConsumerCreateResponse TMultiClusterConsumer::MakeCreateResponse(const TError& description) { + TReadResponse res; + res.MutableError()->CopyFrom(description); + return TConsumerCreateResponse(std::move(res)); +} + +TConsumerMessage TMultiClusterConsumer::MakeResponse(const TError& description) { + TReadResponse res; + res.MutableError()->CopyFrom(description); + return TConsumerMessage(std::move(res)); +} + +TMultiClusterConsumer::TSubconsumerInfo::TSubconsumerInfo(std::shared_ptr<IConsumerImpl> consumer) + : Consumer(std::move(consumer)) + , StartFuture(Consumer->Start()) + , DeadFuture(Consumer->IsDead()) +{ +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h new file mode 100644 index 0000000000..ab8a338fbe --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer.h @@ -0,0 +1,93 @@ +#pragma once +#include "channel_p.h" +#include "scheduler.h" +#include "iconsumer_p.h" +#include "internals.h" + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> + +namespace NPersQueue { + +class TPQLibPrivate; + +class TMultiClusterConsumer: public IConsumerImpl, public std::enable_shared_from_this<TMultiClusterConsumer> { +public: + struct TSubconsumerInfo; + struct TCookieMappingItem; + struct TCookieMapping; + +public: + TMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); + ~TMultiClusterConsumer(); + + NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline) noexcept override; + NThreading::TFuture<TError> IsDead() noexcept override; + + void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; + void Commit(const TVector<ui64>& cookies) noexcept override; + void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; + + void Cancel() override; + +private: + void PatchSettings(); + void StartClusterDiscovery(TInstant deadline); + void OnClusterDiscoveryDone(TInstant deadline); + void OnStartTimeout(); + void OnSubconsumerStarted(size_t subconsumerIndex); + + void CheckReadyResponses(); + bool ProcessReadyResponses(size_t subconsumerIndex); // False if consumer has been destroyed. + void RequestSubconsumers(); // Make requests inflight in every subconsumer equal to our superconsumer inflight. + + bool TranslateConsumerMessage(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); // False if consumer has been destroyed. + bool TranslateConsumerMessageLock(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); + bool TranslateConsumerMessageRelease(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); + bool TranslateConsumerMessageData(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); + bool TranslateConsumerMessageCommit(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); + bool TranslateConsumerMessageError(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); // Calls destroy on consumer + bool TranslateConsumerMessageStatus(NThreading::TPromise<TConsumerMessage>& promise, TConsumerMessage&& subconsumerResponse, size_t subconsumerIndex); + + void Destroy(const TError& description); + void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); + void Destroy(); + + static TError MakeError(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); + static TConsumerCreateResponse MakeCreateResponse(const TError& description); + static TConsumerMessage MakeResponse(const TError& description); + + void ScheduleClusterDiscoveryRetry(TInstant deadline); + +private: + TConsumerSettings Settings; + TIntrusivePtr<ILogger> Logger; + TString SessionId; + + NThreading::TPromise<TConsumerCreateResponse> StartPromise = NThreading::NewPromise<TConsumerCreateResponse>(); + NThreading::TPromise<TError> IsDeadPromise = NThreading::NewPromise<TError>(); + + enum class EState { + Created, // Before Start() was created. + WaitingClusterDiscovery, // After Start() was called and before we received CDS response. + StartingSubconsumers, // After we received CDS response and before consumer actually started. + Working, // When one or more consumers had started. + Dead, + }; + EState State = EState::Created; + + TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback; + TChannelImplPtr ClusterDiscoverer; + TConsumerChannelOverCdsImpl::TResultPtr ClusterDiscoverResult; + size_t ClusterDiscoveryAttemptsDone = 0; + + std::vector<TSubconsumerInfo> Subconsumers; + size_t CurrentSubconsumer = 0; // Index of current subconsumer to take responses from + + THolder<TCookieMapping> CookieMapping; + THashMap<TString, size_t> OldTopicName2Subconsumer; // Used in lock session for requesting partition status. + + std::deque<NThreading::TPromise<TConsumerMessage>> Requests; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp new file mode 100644 index 0000000000..a29a18cbaf --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_consumer_ut.cpp @@ -0,0 +1,227 @@ +#include "multicluster_consumer.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/event.h> + +namespace NPersQueue { + +Y_UNIT_TEST_SUITE(TMultiClusterConsumerTest) { + TConsumerSettings MakeFakeConsumerSettings() { + TConsumerSettings settings; + settings.Topics.push_back("topic"); + settings.ReadFromAllClusterSources = true; + settings.ReadMirroredPartitions = false; + settings.MaxAttempts = 1; // Consumer should fix this setting. Check it. + return settings; + } + + TConsumerSettings MakeConsumerSettings(const SDKTestSetup& setup) { + TConsumerSettings settings = setup.GetConsumerSettings(); + settings.ReadFromAllClusterSources = true; + settings.ReadMirroredPartitions = false; + settings.MaxAttempts = 1; // Consumer should fix this setting. Check it. + return settings; + } + + Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) { + // Test that consumer doesn't hang on till shutdown + TPQLib lib; + lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); + } + + Y_UNIT_TEST(StartedConsumerCanBeDestructed) { + // Test that consumer doesn't hang on till shutdown + TPQLib lib; + auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); + auto isDead = consumer->IsDead(); + consumer->Start(); + consumer = nullptr; + isDead.GetValueSync(); + } + + Y_UNIT_TEST(CommitRightAfterStart) { + SDKTestSetup setup("InfiniteStart"); + TPQLib lib; + auto settings = MakeConsumerSettings(setup); + settings.Topics[0] = "unknown_topic"; + auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); + consumer->Start(); + auto isDead = consumer->IsDead(); + consumer->Commit({42}); + if (GrpcV1EnabledByDefault()) { + UNIT_ASSERT_EQUAL_C(isDead.GetValueSync().GetCode(), NErrorCode::WRONG_COOKIE, isDead.GetValueSync()); + } else { + UNIT_ASSERT_STRINGS_EQUAL_C(isDead.GetValueSync().GetDescription(), "Requesting commit, but consumer is not in working state", isDead.GetValueSync()); + } + } + + Y_UNIT_TEST(GetNextMessageRightAfterStart) { + if (GrpcV1EnabledByDefault()) { + return; + } + SDKTestSetup setup("InfiniteStart"); + TPQLib lib; + auto settings = MakeConsumerSettings(setup); + settings.Topics[0] = "unknown_topic"; + auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); + consumer->Start(); + auto nextMessage = consumer->GetNextMessage(); + UNIT_ASSERT_STRINGS_EQUAL(nextMessage.GetValueSync().Response.GetError().GetDescription(), "Requesting next message, but consumer is not in working state"); + } + + Y_UNIT_TEST(RequestPartitionStatusRightAfterStart) { + if (GrpcV1EnabledByDefault()) { + return; + } + SDKTestSetup setup("InfiniteStart"); + TPQLib lib; + auto settings = MakeConsumerSettings(setup); + settings.Topics[0] = "unknown_topic"; + auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); + consumer->Start(); + auto isDead = consumer->IsDead(); + consumer->RequestPartitionStatus("topic", 42, 42); + UNIT_ASSERT_STRINGS_EQUAL(isDead.GetValueSync().GetDescription(), "Requesting partition status, but consumer is not in working state"); + } + + Y_UNIT_TEST(StartedConsumerCantBeStartedAgain) { + TPQLib lib; + auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); + auto startFuture = consumer->Start(); + auto startFuture2 = consumer->Start(); + UNIT_ASSERT(startFuture2.HasValue()); + UNIT_ASSERT_STRINGS_EQUAL(startFuture2.GetValueSync().Response.GetError().GetDescription(), "Start was already called"); + } + + Y_UNIT_TEST(StartWithFailedCds) { + TPQLib lib; + auto consumer = lib.CreateConsumer(MakeFakeConsumerSettings(), new TCerrLogger(TLOG_DEBUG)); + auto startFuture = consumer->Start(TDuration::MilliSeconds(10)); + if (!GrpcV1EnabledByDefault()) { + UNIT_ASSERT_STRING_CONTAINS(startFuture.GetValueSync().Response.GetError().GetDescription(), " cluster discovery attempts"); + } + auto isDead = consumer->IsDead(); + isDead.Wait(); + } + + Y_UNIT_TEST(StartDeadline) { + if (GrpcV1EnabledByDefault()) { + return; + } + TPQLib lib; + TConsumerSettings settings = MakeFakeConsumerSettings(); + auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); + auto startFuture = consumer->Start(TDuration::MilliSeconds(100)); + UNIT_ASSERT_EQUAL_C(startFuture.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, startFuture.GetValueSync().Response); + } + + Y_UNIT_TEST(StartDeadlineAfterCds) { + SDKTestSetup setup("StartDeadlineAfterCds"); + TPQLib lib; + auto settings = MakeConsumerSettings(setup); + settings.Topics[0] = "unknown_topic"; + auto consumer = lib.CreateConsumer(settings, new TCerrLogger(TLOG_DEBUG)); + auto startFuture = consumer->Start(TDuration::MilliSeconds(1)); + if (!GrpcV1EnabledByDefault()) { + UNIT_ASSERT_STRINGS_EQUAL(startFuture.GetValueSync().Response.GetError().GetDescription(), "Start timeout"); + } + auto isDead = consumer->IsDead(); + isDead.Wait(); + } + + void WorksWithSingleDc(bool oneDcIsUnavailable) { + SDKTestSetup setup("WorksWithSingleDc", false); + if (!oneDcIsUnavailable) { + setup.SetSingleDataCenter(); // By default one Dc is fake in test setup. This call overrides it. + } + setup.Start(); + auto consumer = setup.StartConsumer(MakeConsumerSettings(setup)); + setup.WriteToTopic({"msg1", "msg2"}); + setup.ReadFromTopic({{"msg1", "msg2"}}, true, consumer.Get()); + } + + Y_UNIT_TEST(WorksWithSingleDc) { + WorksWithSingleDc(false); + } + + Y_UNIT_TEST(WorksWithSingleDcWhileSecondDcIsUnavailable) { + WorksWithSingleDc(true); + } + + Y_UNIT_TEST(FailsOnCommittingWrongCookie) { + SDKTestSetup setup("FailsOnCommittingWrongCookie"); + auto consumer = setup.StartConsumer(MakeConsumerSettings(setup)); + setup.WriteToTopic({"msg1", "msg2"}); + auto nextMessage = consumer->GetNextMessage(); + UNIT_ASSERT_EQUAL(nextMessage.GetValueSync().Type, EMT_DATA); + auto isDead = consumer->IsDead(); + consumer->Commit({nextMessage.GetValueSync().Response.GetData().GetCookie() + 1}); + UNIT_ASSERT_STRING_CONTAINS(isDead.GetValueSync().GetDescription(), "Wrong cookie"); + } + + // Test that consumer is properly connected with decompressing consumer. + void Unpacks(bool unpack) { + SDKTestSetup setup(TStringBuilder() << "Unpacks(" << (unpack ? "true)" : "false)")); + setup.WriteToTopic({"msg"}); + auto settings = MakeConsumerSettings(setup); + settings.Unpack = unpack; + auto consumer = setup.StartConsumer(settings); + auto readResponse = consumer->GetNextMessage().GetValueSync(); + UNIT_ASSERT_C(!readResponse.Response.HasError(), readResponse.Response); + UNIT_ASSERT_C(readResponse.Response.HasData(), readResponse.Response); + if (unpack) { + UNIT_ASSERT_EQUAL_C(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), NPersQueueCommon::RAW, readResponse.Response); + UNIT_ASSERT_STRINGS_EQUAL(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg"); + } else { + UNIT_ASSERT_EQUAL_C(readResponse.Response.GetData().GetMessageBatch(0).GetMessage(0).GetMeta().GetCodec(), NPersQueueCommon::GZIP, readResponse.Response); + } + } + + Y_UNIT_TEST(UnpacksIfNeeded) { + Unpacks(true); + } + + Y_UNIT_TEST(DoesNotUnpackIfNeeded) { + Unpacks(false); + } + + Y_UNIT_TEST(WorksWithManyDc) { + SDKTestSetup setup1("WorksWithManyDc1", false); + SDKTestSetup setup2("WorksWithManyDc2"); + SDKTestSetup setup3("WorksWithManyDc3", false); + setup1.AddDataCenter("dc2", setup2); + setup1.AddDataCenter("dc3", setup3); + setup3.GetGrpcServerOptions().SetGRpcShutdownDeadline(TDuration::Seconds(1)); + setup3.Start(); + setup1.Start(); + + // Check that consumer reads from all sources. + setup1.WriteToTopic({"msg1", "msg2"}); + setup2.WriteToTopic({"msg3", "msg4"}); + setup3.WriteToTopic({"msg10", "msg11", "msg12", "msg13", "msg14", "msg15"}); + auto consumer = setup1.StartConsumer(MakeConsumerSettings(setup1)); + setup1.ReadFromTopic({{"msg1", "msg2"}, {"msg3", "msg4"}, {"msg10", "msg11", "msg12", "msg13", "msg14", "msg15"}}, true, consumer.Get()); + + // Turn Dc off and check that consumer works. + setup1.GetLog() << TLOG_INFO << "Turn off DataCenter dc3"; + setup3.ShutdownGRpc(); + setup1.WriteToTopic({"a", "b"}); + setup2.WriteToTopic({"x", "y", "z"}); + setup1.ReadFromTopic({{"a", "b"}, {"x", "y", "z"}}, true, consumer.Get()); + + // Turn Dc on again and check work again. + setup1.GetLog() << TLOG_INFO << "Turn on DataCenter dc3"; + setup3.EnableGRpc(); + setup1.WriteToTopic({"1", "2"}); + setup3.WriteToTopic({"42"}); + setup1.ReadFromTopic({{"1", "2"}, {"42"}}, true, consumer.Get()); + } +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp new file mode 100644 index 0000000000..6f63b0ab4a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.cpp @@ -0,0 +1,423 @@ +#include "multicluster_producer.h" +#include "persqueue_p.h" + +#include <util/string/builder.h> + +#include <algorithm> + +namespace NPersQueue { + +TWeightIndex::TWeightIndex(size_t count) + : Weights(count) + , Index(count) + , IsEnabled(count) + , EnabledCnt(0) +{ + Y_VERIFY(count > 0); +} + +void TWeightIndex::SetWeight(size_t i, unsigned weight) { + const unsigned oldWeight = Weights[i]; + const int diff = static_cast<int>(weight) - static_cast<int>(oldWeight); + if (diff != 0) { + Weights[i] = weight; + if (IsEnabled[i]) { + UpdateIndex(i, diff); + } + } +} + +void TWeightIndex::Enable(size_t i) { + if (!IsEnabled[i]) { + ++EnabledCnt; + IsEnabled[i] = true; + UpdateIndex(i, static_cast<int>(Weights[i])); + } +} + +void TWeightIndex::Disable(size_t i) { + if (IsEnabled[i]) { + --EnabledCnt; + IsEnabled[i] = false; + UpdateIndex(i, -static_cast<int>(Weights[i])); + } +} + +size_t TWeightIndex::Choose(unsigned randomNumber) const { + Y_VERIFY(randomNumber < WeightsSum()); + Y_VERIFY(!Index.empty()); + const auto choice = std::upper_bound(Index.begin(), Index.end(), randomNumber); + Y_VERIFY(choice != Index.end()); + return choice - Index.begin(); +} + +void TWeightIndex::UpdateIndex(size_t i, int weightDiff) { + for (size_t j = i; j < Index.size(); ++j) { + Index[j] += weightDiff; + } +} + +static TProducerCommitResponse MakeErrorCommitResponse(const TString& description, TProducerSeqNo seqNo, const TData& data) { + TWriteResponse res; + res.MutableError()->SetDescription(description); + res.MutableError()->SetCode(NErrorCode::ERROR); + return TProducerCommitResponse{seqNo, data, std::move(res)}; +} + +TMultiClusterProducer::TMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) + : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) + , Settings(settings) + , DeadPromise(NThreading::NewPromise<TError>()) + , StartPromise(NThreading::NewPromise<TProducerCreateResponse>()) + , State(EState::Created) + , Logger(std::move(logger)) + , WeightIndex(Settings.ServerWeights.size()) + , FallbackWeightIndex(Settings.ServerWeights.size()) + , Subproducers(Settings.ServerWeights.size()) + , FallbackSubproducers(Settings.ServerWeights.size()) +{ + if (Settings.ServerWeights.size() <= 1) { + ythrow yexception() << "MultiCluster producer is working with several servers. You'd better to create retrying producer in this case"; + } + if (Settings.MinimumWorkingDcsCount == 0) { + ythrow yexception() << "MinimumWorkingDcsCount can't be zero. We want that you have at least something working!"; + } + if (Settings.SourceIdPrefix.empty()) { + ythrow yexception() << "SourceIdPrefix must be nonempty"; + } + + bool reconnectWarningPrinted = false; + for (TMultiClusterProducerSettings::TServerWeight& w : Settings.ServerWeights) { + if (!w.Weight) { + ythrow yexception() << "Server weight must be greater than zero"; + } + if (!w.ProducerSettings.ReconnectOnFailure) { + w.ProducerSettings.ReconnectOnFailure = true; + if (!reconnectWarningPrinted) { + reconnectWarningPrinted = true; + WARN_LOG("ReconnectOnFailure setting is off. Turning it on for multicluster producer.", Settings.SourceIdPrefix, ""); + } + ythrow yexception() << "You must enable ReconnectOnFailure setting"; + } + if (w.ProducerSettings.MaxAttempts == std::numeric_limits<unsigned>::max()) { + ythrow yexception() << "MaxAttempts must be a finite number"; + } + } + + for (size_t i = 0; i < Settings.ServerWeights.size(); ++i) { + WeightIndex.SetWeight(i, Settings.ServerWeights[i].Weight); + FallbackWeightIndex.SetWeight(i, Settings.ServerWeights[i].Weight); + } +} + +TMultiClusterProducer::~TMultiClusterProducer() { + Destroy("Destructor called"); +} + +void TMultiClusterProducer::StartSubproducer(TInstant deadline, size_t i, bool fallback) { + std::weak_ptr<TMultiClusterProducer> self(shared_from_this()); + auto& sp = GetSubproducers(fallback)[i]; + sp.Producer = CreateSubproducer(Settings.ServerWeights[i].ProducerSettings, i, fallback); + sp.StartFuture = sp.Producer->Start(deadline); + sp.DeadFuture = sp.Producer->IsDead(); + PQLib->Subscribe(sp.StartFuture, + this, + [self, i, fallback](const auto&) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnProducerStarted(i, fallback); + } + }); + PQLib->Subscribe(sp.DeadFuture, + this, + [self, i, fallback](const auto&) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnProducerDead(i, fallback); + } + }); +} + +void TMultiClusterProducer::OnNeedProducerRestart(size_t i, bool fallback) { + if (State == EState::Dead) { + return; + } + const auto deadline = TInstant::Now() + Settings.ServerWeights[i].ProducerSettings.StartSessionTimeout; + StartSubproducer(deadline, i, fallback); +} + +void TMultiClusterProducer::ScheduleProducerRestart(size_t i, bool fallback) { + std::weak_ptr<TMultiClusterProducer> self(shared_from_this()); + PQLib->GetScheduler().Schedule(Settings.ServerWeights[i].ProducerSettings.ReconnectionDelay, + this, + [self, i, fallback] { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnNeedProducerRestart(i, fallback); + } + }); +} + +void TMultiClusterProducer::StartSubproducers(TInstant deadline, bool fallback) { + for (size_t i = 0; i < Settings.ServerWeights.size(); ++i) { + StartSubproducer(deadline, i, fallback); + } +} + +NThreading::TFuture<TProducerCreateResponse> TMultiClusterProducer::Start(TInstant deadline) noexcept { + Y_VERIFY(State == EState::Created); + State = EState::Starting; + StartSubproducers(deadline); + StartSubproducers(deadline, true); + return StartPromise.GetFuture(); +} + +void TMultiClusterProducer::OnProducerStarted(size_t i, bool fallback) { + if (State == EState::Dead) { + return; + } + TSubproducerInfo& info = GetSubproducers(fallback)[i]; + info.StartAnswered = true; + DEBUG_LOG("Start subproducer response: " << info.StartFuture.GetValue().Response, Settings.SourceIdPrefix, ""); + if (info.StartFuture.GetValue().Response.HasError()) { + ERR_LOG("Failed to start subproducer[" << i << "]: " << info.StartFuture.GetValue().Response.GetError().GetDescription(), Settings.SourceIdPrefix, ""); + if (State == EState::Starting && !fallback) { + size_t startedCount = 0; + for (const auto& p : Subproducers) { + startedCount += p.StartAnswered; + } + if (startedCount == Subproducers.size() && WeightIndex.EnabledCount() < Settings.MinimumWorkingDcsCount) { + Destroy("Not enough subproducers started successfully"); + return; + } + } + + ScheduleProducerRestart(i, fallback); + } else { + GetWeightIndex(fallback).Enable(i); + if (State == EState::Starting && !fallback && WeightIndex.EnabledCount() >= Settings.MinimumWorkingDcsCount) { + State = EState::Working; + TWriteResponse response; + response.MutableInit(); + StartPromise.SetValue(TProducerCreateResponse(std::move(response))); + } + if (fallback) { + ResendLastHopeQueue(i); + } + } +} + +void TMultiClusterProducer::OnProducerDead(size_t i, bool fallback) { + if (State == EState::Dead) { + return; + } + + WARN_LOG("Subproducer[" << i << "] is dead: " << GetSubproducers(fallback)[i].DeadFuture.GetValue().GetDescription(), Settings.SourceIdPrefix, ""); + GetWeightIndex(fallback).Disable(i); + if (!fallback && WeightIndex.EnabledCount() < Settings.MinimumWorkingDcsCount) { + Destroy("Not enough subproducers is online"); + return; + } + ScheduleProducerRestart(i, fallback); +} + +void TMultiClusterProducer::Destroy(const TString& description) { + TError error; + error.SetCode(NErrorCode::ERROR); + error.SetDescription(description); + Destroy(error); +} + +void TMultiClusterProducer::DestroyWrites(const TError& error, TIntrusiveListWithAutoDelete<TWriteInfo, TDelete>& pendingWrites) { + for (TWriteInfo& wi : pendingWrites) { + TWriteResponse response; + response.MutableError()->CopyFrom(error); + wi.ResponsePromise.SetValue(TProducerCommitResponse(0, std::move(wi.Data), std::move(response))); + } + pendingWrites.Clear(); +} + +void TMultiClusterProducer::Destroy(const TError& error) { + if (State == EState::Dead) { + return; + } + + DEBUG_LOG("Destroying multicluster producer. Reason: " << error, Settings.SourceIdPrefix, ""); + const EState prevState = State; + State = EState::Dead; + SubscribeDestroyed(); + if (prevState == EState::Starting) { + TWriteResponse response; + response.MutableError()->CopyFrom(error); + StartPromise.SetValue(TProducerCreateResponse(std::move(response))); + } + for (size_t i = 0; i < Subproducers.size(); ++i) { + Subproducers[i].Producer = nullptr; + FallbackSubproducers[i].Producer = nullptr; + DestroyWrites(error, FallbackSubproducers[i].LastHopePendingWrites); + } + DestroyWrites(error, PendingWrites); + DeadPromise.SetValue(error); + + DestroyPQLibRef(); +} + +void TMultiClusterProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept { + return promise.SetValue(MakeErrorCommitResponse("MultiCluster producer doesn't allow to write with explicitly specified seq no.", seqNo, data)); +} + +void TMultiClusterProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept { + Y_VERIFY(data.IsEncoded()); + + if (State != EState::Working) { + TWriteResponse res; + res.MutableError()->SetDescription(TStringBuilder() << "producer is " << (State == EState::Dead ? "dead" : "not ready") << ". Marker# PQLib01"); + res.MutableError()->SetCode(NErrorCode::ERROR); + promise.SetValue(TProducerCommitResponse{0, std::move(data), std::move(res)}); + return; + } + + TWriteInfo* wi = new TWriteInfo(); + PendingWrites.PushBack(wi); + + const size_t i = WeightIndex.RandomChoose(); + wi->Data = std::move(data); + wi->ResponsePromise = promise; + DelegateWrite(wi, i); +} + +void TMultiClusterProducer::DelegateWrite(TWriteInfo* wi, size_t i, bool fallback) { + if (wi->History.empty() || wi->History.back() != i) { + wi->History.push_back(i); + } + DEBUG_LOG("Delegate" << (fallback ? " fallback" : "") << " write to subproducer[" << i << "]", Settings.SourceIdPrefix, ""); + wi->ResponseFuture = GetSubproducers(fallback)[i].Producer->Write(wi->Data); + wi->WaitingCallback = true; + + std::weak_ptr<TMultiClusterProducer> self(shared_from_this()); + PQLib->Subscribe(wi->ResponseFuture, + this, + [self, wi](const auto&) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnWriteResponse(wi); + } + }); +} + +void TMultiClusterProducer::ResendLastHopeQueue(size_t i) { + TSubproducerInfo& info = FallbackSubproducers[i]; + if (!info.LastHopePendingWrites.Empty()) { + for (TWriteInfo& wi : info.LastHopePendingWrites) { + if (!wi.WaitingCallback) { + DelegateWrite(&wi, i, true); + } + } + } +} + +void TMultiClusterProducer::OnWriteResponse(TWriteInfo* wi) { + if (State == EState::Dead) { + return; + } + + Y_VERIFY(wi->WaitingCallback); + Y_VERIFY(!wi->History.empty()); + + DEBUG_LOG("Write response: " << wi->ResponseFuture.GetValue().Response, Settings.SourceIdPrefix, ""); + wi->WaitingCallback = false; + if (wi->ResponseFuture.GetValue().Response.HasAck()) { + wi->ResponsePromise.SetValue(wi->ResponseFuture.GetValue()); + wi->Unlink(); + delete wi; + } else if (wi->ResponseFuture.GetValue().Response.HasError()) { + // retry write to other dc + if (wi->LastHope) { + const size_t i = wi->History.back(); + if (FallbackWeightIndex.Enabled(i)) { + DelegateWrite(wi, i, true); + } // else we will retry when this producer will become online. + return; + } + TWeightIndex index = FallbackWeightIndex; + for (size_t i : wi->History) { + index.Disable(i); + } + + // choose new fallback dc + size_t i; + if (index.EnabledCount() == 0) { + i = wi->History.back(); + } else { + i = index.RandomChoose(); + } + + // define whether this dc is the last hope + const bool lastHope = index.EnabledCount() <= 1; + if (lastHope) { + wi->LastHope = true; + wi->Unlink(); + FallbackSubproducers[i].LastHopePendingWrites.PushBack(wi); + } + + DEBUG_LOG("Retrying write to subproducer[" << i << "], lastHope: " << lastHope, Settings.SourceIdPrefix, ""); + DelegateWrite(wi, i, true); + } else { // incorrect structure of response + Y_VERIFY(false); + } +} + +NThreading::TFuture<TError> TMultiClusterProducer::IsDead() noexcept { + return DeadPromise.GetFuture(); +} + +TString TMultiClusterProducer::MakeSourceId(const TStringBuf& prefix, const TStringBuf& srcSourceId, + size_t producerIndex, bool fallback) { + TStringBuilder ret; + ret << prefix << "-" << producerIndex; + if (!srcSourceId.empty()) { + ret << "-"sv << srcSourceId; + } + if (fallback) { + ret << "-multicluster-fallback"sv; + } else { + ret << "-multicluster"sv; + } + return std::move(ret); +} + +std::shared_ptr<IProducerImpl> TMultiClusterProducer::CreateSubproducer(const TProducerSettings& srcSettings, + size_t index, bool fallback) { + TProducerSettings settings = srcSettings; + settings.SourceId = MakeSourceId(Settings.SourceIdPrefix, srcSettings.SourceId, index, fallback); + return PQLib->CreateRawRetryingProducer(settings, DestroyEventRef, Logger); +} + +void TMultiClusterProducer::SubscribeDestroyed() { + NThreading::TPromise<void> promise = ProducersDestroyed; + auto handler = [promise](const auto&) mutable { + promise.SetValue(); + }; + std::vector<NThreading::TFuture<void>> futures; + futures.reserve(Subproducers.size() * 2 + 1); + futures.push_back(DestroyedPromise.GetFuture()); + for (size_t i = 0; i < Subproducers.size(); ++i) { + if (Subproducers[i].Producer) { + futures.push_back(Subproducers[i].Producer->Destroyed()); + } + if (FallbackSubproducers[i].Producer) { + futures.push_back(FallbackSubproducers[i].Producer->Destroyed()); + } + } + WaitExceptionOrAll(futures).Subscribe(handler); +} + +NThreading::TFuture<void> TMultiClusterProducer::Destroyed() noexcept { + return ProducersDestroyed.GetFuture(); +} + +void TMultiClusterProducer::Cancel() { + Destroy(GetCancelReason()); +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h new file mode 100644 index 0000000000..16fcfb6b11 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer.h @@ -0,0 +1,147 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include "internals.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include "channel.h" +#include "iproducer_p.h" + +#include <library/cpp/threading/future/future.h> + +#include <util/generic/intrlist.h> +#include <util/random/random.h> + +#include <deque> + +namespace NPersQueue { + +// Structure to perform choose by weights +class TWeightIndex { +public: + // Creates zero-weighted disabled index + explicit TWeightIndex(size_t count); + + void SetWeight(size_t i, unsigned weight); + + void Enable(size_t i); + void Disable(size_t i); + + // Chooses by random number in interval [0, WeightsSum()) + size_t Choose(unsigned randomNumber) const; + + size_t RandomChoose() const { + return Choose(RandomNumber<unsigned>(WeightsSum())); + } + + size_t EnabledCount() const { + return EnabledCnt; + } + + bool Enabled(size_t i) const { + return i < IsEnabled.size() ? IsEnabled[i] : false; + } + + unsigned WeightsSum() const { + return Index.back(); + } + +private: + void UpdateIndex(size_t i, int weightDiff); + +private: + std::vector<unsigned> Weights; + std::vector<unsigned> Index; // structure for choosing. Index[i] is sum of enabled weights with index <= i. + std::vector<bool> IsEnabled; + size_t EnabledCnt; +}; + +class TMultiClusterProducer: public IProducerImpl, public std::enable_shared_from_this<TMultiClusterProducer> { +public: + TMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); + ~TMultiClusterProducer(); + + NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; + + using IProducerImpl::Write; + + // Forbidden write + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TProducerSeqNo seqNo, TData data) noexcept override; + + // Allowed write + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; + + NThreading::TFuture<TError> IsDead() noexcept override; + + NThreading::TFuture<void> Destroyed() noexcept override; + + void Cancel() override; + +private: + struct TWriteInfo: public TIntrusiveListItem<TWriteInfo> { + NThreading::TPromise<TProducerCommitResponse> ResponsePromise; + NThreading::TFuture<TProducerCommitResponse> ResponseFuture; + bool WaitingCallback = false; + std::vector<size_t> History; // Number of DCs with failed or pending writes + TData Data; + bool LastHope = false; // This DC is the last hope: retry till success. + // If this flag is set, this write info is in the local subproducer queue. + }; + + struct TSubproducerInfo { + size_t Number = std::numeric_limits<size_t>::max(); + std::shared_ptr<IProducerImpl> Producer; + NThreading::TFuture<TProducerCreateResponse> StartFuture; + NThreading::TFuture<TError> DeadFuture; + TIntrusiveListWithAutoDelete<TWriteInfo, TDelete> LastHopePendingWrites; // Local queue with writes only to this DC. + bool StartAnswered = false; + }; + +private: + std::shared_ptr<IProducerImpl> CreateSubproducer(const TProducerSettings& settings, size_t index, bool fallback = false); + + std::vector<TSubproducerInfo>& GetSubproducers(bool fallback = false) { + return fallback ? FallbackSubproducers : Subproducers; + } + + TWeightIndex& GetWeightIndex(bool fallback = false) { + return fallback ? FallbackWeightIndex : WeightIndex; + } + + static TString MakeSourceId(const TStringBuf& prefix, const TStringBuf& srcSourceId, + size_t producerIndex, bool fallback); + void StartSubproducers(TInstant deadline, bool fallback = false); + void StartSubproducer(TInstant deadline, size_t i, bool fallback = false); + void OnProducerStarted(size_t i, bool fallback); + void OnProducerDead(size_t i, bool fallback); + void OnWriteResponse(TWriteInfo* wi); + void ScheduleProducerRestart(size_t i, bool fallback); + void OnNeedProducerRestart(size_t i, bool fallback); // scheduler action + void Destroy(const TString& description); + void Destroy(const TError& error); + void DestroyWrites(const TError& error, TIntrusiveListWithAutoDelete<TWriteInfo, TDelete>& pendingWrites); + void DelegateWrite(TWriteInfo* wi, size_t i, bool fallback = false); + void ResendLastHopeQueue(size_t i); + void SubscribeDestroyed(); + +protected: + TMultiClusterProducerSettings Settings; + NThreading::TPromise<TError> DeadPromise; + NThreading::TPromise<TProducerCreateResponse> StartPromise; + enum class EState { + Created, + Starting, + Working, + Dead, + }; + EState State; + TIntrusivePtr<ILogger> Logger; + TWeightIndex WeightIndex; + TWeightIndex FallbackWeightIndex; + std::vector<TSubproducerInfo> Subproducers; + std::vector<TSubproducerInfo> FallbackSubproducers; + TIntrusiveListWithAutoDelete<TWriteInfo, TDelete> PendingWrites; + NThreading::TPromise<void> ProducersDestroyed = NThreading::NewPromise<void>(); +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp new file mode 100644 index 0000000000..6942681e6a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/multicluster_producer_ut.cpp @@ -0,0 +1,360 @@ +#include "multicluster_producer.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/event.h> + +#include <atomic> + +namespace NPersQueue { + +Y_UNIT_TEST_SUITE(TWeightIndexTest) { + Y_UNIT_TEST(Enables) { + TWeightIndex wi(3); + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); + + wi.SetWeight(0, 1); + wi.SetWeight(1, 2); + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 0); + + wi.Disable(1); + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); + UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 0); + + wi.Enable(0); + wi.Enable(0); // no changes + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1); + + wi.Enable(1); + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 3); + + wi.Disable(1); + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1); + + wi.Disable(1); // no changes + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 1); + UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 1); + } + + Y_UNIT_TEST(Chooses) { + const size_t weightsCount = 3; + TWeightIndex wi(weightsCount); + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), 0); + unsigned weights[weightsCount] = {1, 2, 3}; + for (size_t i = 0; i < weightsCount; ++i) { + wi.SetWeight(i, weights[i]); + wi.Enable(i); + } + UNIT_ASSERT_VALUES_EQUAL(wi.EnabledCount(), weightsCount); + UNIT_ASSERT_VALUES_EQUAL(wi.WeightsSum(), 6); + unsigned cnt[weightsCount] = {}; + for (size_t i = 0; i < wi.WeightsSum(); ++i) { + const size_t choice = wi.Choose(i); + UNIT_ASSERT(choice < weightsCount); // it is index + ++cnt[choice]; + } + for (size_t i = 0; i < weightsCount; ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(weights[i], cnt[i], "i: " << i); + } + } +} + +Y_UNIT_TEST_SUITE(TMultiClusterProducerTest) { + Y_UNIT_TEST(NotStartedProducerCanBeDestructed) { + // Test that producer doesn't hang on till shutdown + TPQLib lib; + TMultiClusterProducerSettings settings; + settings.SourceIdPrefix = "prefix"; + settings.ServerWeights.resize(2); + + { + auto& w = settings.ServerWeights[0]; + w.Weight = 100500; + auto& s = w.ProducerSettings; + s.ReconnectOnFailure = true; + s.Server = TServerSetting{"localhost"}; + s.Topic = "topic1"; + s.MaxAttempts = 10; + } + + { + auto& w = settings.ServerWeights[1]; + w.Weight = 1; + auto& s = w.ProducerSettings; + s.ReconnectOnFailure = true; + s.Server = TServerSetting{"localhost"}; + s.Topic = "topic2"; + s.MaxAttempts = 10; + } + lib.CreateMultiClusterProducer(settings, {}, false); + } + + static void InitServer(TTestServer& testServer) { + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2); + + testServer.WaitInit("topic1"); + } + + static void ConnectsToMinOnlineDcs(size_t min) { + return; // Test is ignored. FIX: KIKIMR-7886 + + TTestServer testServer(false); + InitServer(testServer); + + TMultiClusterProducerSettings settings; + settings.SourceIdPrefix = "prefix"; + settings.ServerWeights.resize(2); + settings.MinimumWorkingDcsCount = min; + + { + auto& w = settings.ServerWeights[0]; + w.Weight = 100500; + auto& s = w.ProducerSettings; + s.ReconnectOnFailure = true; + s.Server = TServerSetting{"localhost", testServer.PortManager->GetPort()}; // port with nothing up + s.Topic = "topic1"; + s.ReconnectionDelay = TDuration::MilliSeconds(1); + s.MaxAttempts = 10; + } + + { + auto& w = settings.ServerWeights[1]; + w.Weight = 1; + auto& s = w.ProducerSettings; + s.ReconnectOnFailure = true; + s.Server = TServerSetting{"localhost", testServer.GrpcPort}; + s.Topic = "topic1"; + s.MaxAttempts = 10; + } + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLib lib; + auto producer = lib.CreateMultiClusterProducer(settings, logger, false); + auto start = producer->Start(); + + if (min == 1) { + UNIT_ASSERT_C(!start.GetValueSync().Response.HasError(), start.GetValueSync().Response); + } else { + UNIT_ASSERT_C(start.GetValueSync().Response.HasError(), start.GetValueSync().Response); + } + + DestroyAndWait(producer); + } + + Y_UNIT_TEST(ConnectsToMinOnlineDcs) { + ConnectsToMinOnlineDcs(1); + } + + Y_UNIT_TEST(FailsToConnectToMinOnlineDcs) { + return; // Test is ignored. FIX: KIKIMR-7886 + + ConnectsToMinOnlineDcs(2); + } + + TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { + TConsumerSettings consumerSettings; + consumerSettings.ClientId = "user"; + consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + consumerSettings.Topics.push_back("topic1"); + consumerSettings.CommitsDisabled = true; + return consumerSettings; + } + + void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { + const TProducerCommitResponse& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); + } + + void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); + } + + std::pair<size_t, size_t> ReadNMessagesAndDestroyConsumers(THolder<IConsumer> consumer1, THolder<IConsumer> consumer2, size_t n) { + std::atomic<size_t> readsDone(0), reads1(0), reads2(0); + TSystemEvent ev; + std::pair<std::atomic<size_t>*, IConsumer*> infos[] = { + std::make_pair(&reads1, consumer1.Get()), + std::make_pair(&reads2, consumer2.Get()) + }; + for (size_t i = 0; i < n; ++i) { + for (auto& info : infos) { + auto* reads = info.first; + info.second->GetNextMessage().Subscribe([&, reads](const NThreading::TFuture<TConsumerMessage>& respFuture) { + ++(*reads); + const size_t newReadsDone = ++readsDone; + if (newReadsDone == n) { + ev.Signal(); + } + if (newReadsDone <= n) { + AssertReadValid(respFuture); + } + }); + } + } + ev.Wait(); + consumer1 = nullptr; + consumer2 = nullptr; + return std::pair<size_t, size_t>(reads1, reads2); + } + + Y_UNIT_TEST(WritesInSubproducers) { + return; // Test is ignored. FIX: KIKIMR-7886 + + TTestServer testServer1(false); + InitServer(testServer1); + + TTestServer testServer2(false); + InitServer(testServer2); + + TMultiClusterProducerSettings settings; + settings.SourceIdPrefix = "producer"; + settings.ServerWeights.resize(2); + settings.MinimumWorkingDcsCount = 1; + + { + auto& w = settings.ServerWeights[0]; + w.Weight = 1; + auto& s = w.ProducerSettings; + s.ReconnectOnFailure = true; + s.Server = TServerSetting{"localhost", testServer1.GrpcPort}; + s.Topic = "topic1"; + s.SourceId = "src0"; + s.MaxAttempts = 5; + s.ReconnectionDelay = TDuration::MilliSeconds(30); + } + + { + auto& w = settings.ServerWeights[1]; + w.Weight = 1; + auto& s = w.ProducerSettings; + s.ReconnectOnFailure = true; + s.Server = TServerSetting{"localhost", testServer2.GrpcPort}; + s.Topic = "topic1"; + s.SourceId = "src1"; + s.MaxAttempts = 2; + s.ReconnectionDelay = TDuration::MilliSeconds(30); + } + + TPQLibSettings pqLibSettings; + pqLibSettings.ThreadsCount = 1; + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLib lib(pqLibSettings); + auto producer = lib.CreateMultiClusterProducer(settings, logger, false); + auto startProducerResult = producer->Start(); + UNIT_ASSERT_C(!startProducerResult.GetValueSync().Response.HasError(), startProducerResult.GetValueSync().Response); + + auto consumer1 = lib.CreateConsumer(MakeConsumerSettings(testServer1), logger, false); + UNIT_ASSERT(!consumer1->Start().GetValueSync().Response.HasError()); + + auto consumer2 = lib.CreateConsumer(MakeConsumerSettings(testServer2), logger, false); + UNIT_ASSERT(!consumer2->Start().GetValueSync().Response.HasError()); + + const size_t writes = 100; + for (size_t i = 0; i < writes; ++i) { + auto write = producer->Write(TString(TStringBuilder() << "blob_1_" << i)); + AssertWriteValid(write); + } + + const std::pair<size_t, size_t> msgs = ReadNMessagesAndDestroyConsumers(std::move(consumer1), std::move(consumer2), writes); + UNIT_ASSERT_C(msgs.first > 30, "msgs.first = " << msgs.first); + UNIT_ASSERT_C(msgs.second > 30, "msgs.second = " << msgs.second); + + consumer1 = lib.CreateConsumer(MakeConsumerSettings(testServer1), logger, false); + UNIT_ASSERT(!consumer1->Start().GetValueSync().Response.HasError()); + + testServer2.ShutdownServer(); + //testServer2.CleverServer->ShutdownGRpc(); + + // Assert that all writes go in 1st DC + for (size_t i = 0; i < writes; ++i) { + auto write = producer->Write(TString(TStringBuilder() << "blob_2_" << i)); + auto read = consumer1->GetNextMessage(); + AssertWriteValid(write); + AssertReadValid(read); + } + + // Assert that all futures will be signalled after producer's death + std::vector<NThreading::TFuture<TProducerCommitResponse>> writesFutures(writes); + for (size_t i = 0; i < writes; ++i) { + writesFutures[i] = producer->Write(TString(TStringBuilder() << "blob_3_" << i)); + } + + auto isDead = producer->IsDead(); + producer = nullptr; + for (auto& f : writesFutures) { + f.GetValueSync(); + } + isDead.GetValueSync(); + + DestroyAndWait(consumer1); + DestroyAndWait(consumer2); + } + + Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { + return; // Test is ignored. FIX: KIKIMR-7886 + + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(); + + const size_t partitions = 1; + testServer.AnnoyingClient->InitRoot(); + testServer.AnnoyingClient->InitDCs(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + testServer.AnnoyingClient->InitSourceIds(); + + testServer.WaitInit("topic1"); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLibSettings pqLibSettings; + pqLibSettings.DefaultLogger = logger; + THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); + + TMultiClusterProducerSettings settings; + settings.SourceIdPrefix = "producer"; + settings.ServerWeights.resize(2); + settings.MinimumWorkingDcsCount = 1; + + for (auto& w : settings.ServerWeights) { + w.Weight = 1; + auto& s = w.ProducerSettings; + s.ReconnectOnFailure = true; + s.Server = TServerSetting{"localhost", testServer.GrpcPort}; + s.Topic = "topic1"; + s.SourceId = "src0"; + s.MaxAttempts = 5; + s.ReconnectionDelay = TDuration::MilliSeconds(30); + } + + auto producer = PQLib->CreateMultiClusterProducer(settings, logger, false); + UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); + auto isDead = producer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + auto write1 = producer->Write(1, TString("blob1")); + auto write2 = producer->Write(2, TString("blob2")); + + PQLib = nullptr; + + UNIT_ASSERT(write1.HasValue()); + UNIT_ASSERT(write2.HasValue()); + + auto write3 = producer->Write(3, TString("blob3")); + UNIT_ASSERT(write3.HasValue()); + UNIT_ASSERT(write3.GetValue().Response.HasError()); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp new file mode 100644 index 0000000000..cc1f46d835 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue.cpp @@ -0,0 +1,540 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include "internals.h" +#include "persqueue_p.h" +#include "compat_producer.h" +#include "consumer.h" +#include "multicluster_consumer.h" +#include "producer.h" +#include "retrying_consumer.h" +#include "retrying_producer.h" +#include "multicluster_producer.h" +#include "processor.h" +#include "compressing_producer.h" +#include "decompressing_consumer.h" +#include "local_caller.h" +#include "ydb_sdk_consumer.h" + +#include <util/digest/numeric.h> +#include <util/system/thread.h> + +namespace NPersQueue { + + +TString TPQLib::GetUserAgent() const { + return Impl->GetUserAgent(); +} + +void TPQLib::SetUserAgent(const TString& userAgent) { + Impl->SetUserAgent(userAgent); +} + +TPQLib::TPQLib(const TPQLibSettings& settings) + : Impl(new TPQLibPrivate(settings)) +{} + +TPQLib::~TPQLib() { + AtomicSet(Alive, 0); + Impl->CancelObjectsAndWait(); +} + +#define CHECK_PQLIB_ALIVE() Y_VERIFY(AtomicGet(Alive), "Attempt to use PQLib after/during its destruction.") + +THolder<IProducer> TPQLib::CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { + CHECK_PQLIB_ALIVE(); + return MakeHolder<TPublicProducer>(Impl->CreateMultiClusterProducer(settings, deprecated, std::move(logger))); +} + +THolder<IProducer> TPQLib::CreateProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { + CHECK_PQLIB_ALIVE(); + auto producer = settings.ReconnectOnFailure ? + Impl->CreateRetryingProducer(settings, deprecated, std::move(logger)) : + Impl->CreateProducer(settings, deprecated, std::move(logger)); + return MakeHolder<TPublicProducer>(std::move(producer)); +} + +template <class TConsumerOrProducerSettings> +NYdb::NPersQueue::TPersQueueClient& TPQLibPrivate::GetOrCreatePersQueueClient(const TConsumerOrProducerSettings& settings, const TIntrusivePtr<ILogger>& logger) { + with_lock(Lock) { + if (!CompressExecutor) { + CompressExecutor = NYdb::NPersQueue::CreateThreadPoolExecutorAdapter(CompressionPool); + HandlersExecutor = NYdb::NPersQueue::CreateThreadPoolExecutor(1); + } + TWrapperKey key{settings.Server, settings.CredentialsProvider}; + auto it = Wrappers.find(key); + if (Wrappers.end() == it) { + CreateWrapper(key, logger); + it = Wrappers.find(key); + } + return *it->second.PersQueueClient; + } +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateNewConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger) +{ + if (Settings.EnableGRpcV1 && settings.Server.UseLogbrokerCDS != EClusterDiscoveryUsageMode::DontUse && !settings.CommitsDisabled && settings.Unpack) { + auto ret = std::make_shared<TLocalConsumerImplCaller<TYdbSdkCompatibilityConsumer>>(settings, + GetSelfRefsAreDeadEvent(false), + this, + logger, + GetOrCreatePersQueueClient(settings, logger)); + ret->Init(); + AddToDestroySet(ret); + return ret; + } + return nullptr; +} + +THolder<IConsumer> TPQLib::CreateConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { + CHECK_PQLIB_ALIVE(); + return MakeHolder<TPublicConsumer>(Impl->CreateConsumer(settings, deprecated, std::move(logger))); +} + +THolder<IProcessor> TPQLib::CreateProcessor(const TProcessorSettings& settings, TIntrusivePtr<ILogger> logger, bool deprecated) { + CHECK_PQLIB_ALIVE(); + return MakeHolder<TPublicProcessor>(Impl->CreateProcessor(settings, deprecated, std::move(logger))); +} + +void TPQLib::SetLogger(TIntrusivePtr<ILogger> logger) { + CHECK_PQLIB_ALIVE(); + Impl->SetLogger(std::move(logger)); +} + +#undef CHECK_PQLIB_ALIVE + +static void DoExecute(std::shared_ptr<grpc::CompletionQueue> cq, std::shared_ptr<std::atomic<bool>> shuttingDown) { + TThread::SetCurrentThreadName("pqlib_grpc_thr"); + void* tag; + bool ok; + while (cq->Next(&tag, &ok)) { + IQueueEvent* const ev(static_cast<IQueueEvent*>(tag)); + if (shuttingDown->load() || !ev->Execute(ok)) { + ev->DestroyRequest(); + } + } +} + +TPQLibPrivate::TPQLibPrivate(const TPQLibSettings& settings) + : Settings(settings) + , CQ(std::make_shared<grpc::CompletionQueue>()) + , Scheduler(MakeHolder<TScheduler>(this)) + , Logger(settings.DefaultLogger) +{ + Y_VERIFY(Settings.ThreadsCount > 0); + Y_VERIFY(Settings.GRpcThreads > 0); + Y_VERIFY(Settings.CompressionPoolThreads > 0); + CompressionPool = std::make_shared<TThreadPool>(); + CompressionPool->Start(Settings.CompressionPoolThreads); + QueuePool.Start(Settings.ThreadsCount); + GRpcThreads.reserve(Settings.GRpcThreads); + for (size_t i = 0; i < Settings.GRpcThreads; ++i) { + GRpcThreads.emplace_back([cq = CQ, shuttingDown = ShuttingDown](){ DoExecute(cq, shuttingDown); }); + } + CreateSelfRefsAreDeadPtr(); +} + +TPQLibPrivate::~TPQLibPrivate() { + DEBUG_LOG("Destroying PQLib. Destroying scheduler.", "", ""); + Scheduler.Reset(); + + DEBUG_LOG("Destroying PQLib. Set stop flag.", "", ""); + Stop(); + + if (CQ) { + DEBUG_LOG("Destroying PQLib. Shutting down completion queue.", "", ""); + CQ->Shutdown(); + } + + if (!AtomicGet(HasDeprecatedObjects)) { + DEBUG_LOG("Destroying PQLib. Joining threads.", "", ""); + for (std::thread& thread : GRpcThreads) { + thread.join(); + } + } else { + DEBUG_LOG("Destroying PQLib. Detaching threads.", "", ""); + for (std::thread& thread : GRpcThreads) { + thread.detach(); + } + } + + if (YdbDriver) { + YdbDriver->Stop(); + YdbDriver = nullptr; + } + + DEBUG_LOG("Destroying PQLib. Stopping compression pool.", "", ""); + CompressionPool->Stop(); + + DEBUG_LOG("Destroying PQLib. Stopping queue pool.", "", ""); + QueuePool.Stop(); +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + std::shared_ptr<IProducerImpl> res; + auto rawProducerImpl = CreateRawProducerImpl<TProducer>(settings, refsAreDeadPtr, ChooseLogger(logger)); + rawProducerImpl->SetChannel(channelInfo); + res = std::move(rawProducerImpl); + return res; +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + TChannelHolder t = CreateChannel(settings, ChooseLogger(logger)); + std::shared_ptr<IProducerImpl> res; + + auto rawProducerImpl = CreateRawProducerImpl<TProducer>(settings, refsAreDeadPtr, ChooseLogger(logger)); + rawProducerImpl->SetChannel(t); + res = std::move(rawProducerImpl); + return res; +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create producer", settings.SourceId, ""); + std::shared_ptr<IProducerImpl> rawProducer = CreateRawProducer(settings, channelInfo, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); + compressing->Init(); + if (!deprecated) { + AddToDestroySet(compressing); + } + return compressing; +} + +class TLoggerWrapper : public TLogBackend { +public: + TLoggerWrapper(TIntrusivePtr<ILogger> logger) noexcept + : Logger(logger) + {} + + ~TLoggerWrapper() {} + + void WriteData(const TLogRecord& rec) + { + Logger->Log(TString(rec.Data, rec.Len), "", "", (int)rec.Priority); + } + + void ReopenLog() {} + +private: + TIntrusivePtr<ILogger> Logger; +}; + + +void TPQLibPrivate::CreateWrapper(const TWrapperKey& key, TIntrusivePtr<ILogger> logger) { + auto& wrapper = Wrappers[key]; + const auto& settings = key.Settings; + if (!YdbDriver) { + NYdb::TDriverConfig config; + + config.SetEndpoint(TStringBuilder() << settings.Address << ":" << settings.Port) //TODO BAD + .SetDatabase(settings.Database.empty() ? "/Root" : settings.Database) + .SetNetworkThreadsNum(Settings.GRpcThreads) + .SetClientThreadsNum(Settings.ThreadsCount) + .SetGRpcKeepAliveTimeout(TDuration::Seconds(90)) + .SetGRpcKeepAlivePermitWithoutCalls(true) + .SetDiscoveryMode(NYdb::EDiscoveryMode::Async); + if (logger) + config.SetLog(MakeHolder<TLoggerWrapper>(logger)); + if (settings.UseSecureConnection) + config.UseSecureConnection(settings.CaCert); + + YdbDriver = MakeHolder<NYdb::TDriver>(config); + } + + NYdb::NPersQueue::TPersQueueClientSettings pqSettings; + pqSettings.DefaultCompressionExecutor(CompressExecutor) + .DefaultHandlersExecutor(HandlersExecutor) + .ClusterDiscoveryMode(NYdb::NPersQueue::EClusterDiscoveryMode::Auto) + .DiscoveryEndpoint(TStringBuilder() << settings.Address << ":" << settings.Port) + .Database(settings.Database); + if (key.Provider) { + pqSettings.CredentialsProviderFactory(std::shared_ptr<NYdb::ICredentialsProviderFactory>(new TCredentialsProviderFactoryWrapper(key.Provider))); + } + wrapper.PersQueueClient = MakeHolder<NYdb::NPersQueue::TPersQueueClient>(*YdbDriver, pqSettings); +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateNewProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger) +{ + if (Settings.EnableGRpcV1) { + auto ret = std::make_shared<TLocalProducerImplCaller<TYdbSdkCompatibilityProducer>>(settings, + GetOrCreatePersQueueClient(settings, logger), + GetSelfRefsAreDeadEvent(false), + this); + ret->Init(); + AddToDestroySet(ret); + return ret; + } + return nullptr; +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) +{ + auto newP = CreateNewProducer(settings, ChooseLogger(logger)); + if (newP) return newP; + + DEBUG_LOG("Create producer", settings.SourceId, ""); + std::shared_ptr<IProducerImpl> rawProducer = CreateRawProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); + compressing->Init(); + if (!deprecated) { + AddToDestroySet(compressing); + } + return compressing; +} + +template<typename TRawProducerImpl> +std::shared_ptr<TRawProducerImpl> TPQLibPrivate::CreateRawProducerImpl(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create raw producer", settings.SourceId, ""); + const bool addToDestroySet = refsAreDeadPtr != nullptr; + NThreading::TPromise<TProducerCreateResponse> promise = NThreading::NewPromise<TProducerCreateResponse>(); + auto ret = std::make_shared<TLocalProducerImplCaller<TRawProducerImpl>>(settings, CQ, promise, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); + ret->Init(); + if (addToDestroySet) { + AddToDestroySet(ret); + } + return ret; +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + auto t = CreateChannel(settings.Server, settings.CredentialsProvider, ChooseLogger(logger), settings.PreferLocalProxy); + auto res = CreateRawConsumerImpl(settings, std::move(refsAreDeadPtr), ChooseLogger(logger)); + res->SetChannel(t); + return std::move(res); +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + auto res = CreateRawConsumerImpl(settings, refsAreDeadPtr, ChooseLogger(logger)); + res->SetChannel(channelInfo); + return std::move(res); +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create multicluster consumer", "", ""); + const bool addToDestroySet = refsAreDeadPtr != nullptr; + auto consumer = std::make_shared<TLocalConsumerImplCaller<TMultiClusterConsumer>>(settings, std::move(refsAreDeadPtr), this, logger); + consumer->Init(); + if (addToDestroySet) { + AddToDestroySet(consumer); + } + return std::move(consumer); +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) { + const bool addToDestroySet = refsAreDeadPtr != nullptr; + std::shared_ptr<IConsumerImpl> consumer = std::make_shared<TLocalConsumerImplCaller<TDecompressingConsumer>>(std::move(subconsumer), settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); + consumer->Init(); + if (addToDestroySet) { + AddToDestroySet(consumer); + } + return consumer; +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateRawRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) { + const bool addToDestroySet = refsAreDeadPtr != nullptr; + std::shared_ptr<IConsumerImpl> consumer = std::make_shared<TLocalConsumerImplCaller<TRetryingConsumer>>(settings, std::move(refsAreDeadPtr), this, logger); + consumer->Init(); + if (addToDestroySet) { + AddToDestroySet(consumer); + } + return consumer; +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateConsumer(const TConsumerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) +{ + if (auto newConsumer = CreateNewConsumer(settings, ChooseLogger(logger))) { + return newConsumer; + } + + DEBUG_LOG("Create consumer", "", ""); + // Create raw consumer. + std::shared_ptr<IConsumerImpl> consumer; + if (settings.ReadFromAllClusterSources) { + if (settings.ReadMirroredPartitions) { + ythrow yexception() << "Can't create consumer with both ReadMirroredPartitions and ReadFromAllClusterSources options"; + } + if (settings.Server.UseLogbrokerCDS == EClusterDiscoveryUsageMode::DontUse) { + ythrow yexception() << "Can't create consumer with ReadFromAllClusterSources option, but without using cluster discovery"; + } + consumer = CreateRawMultiClusterConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + } else if (settings.ReconnectOnFailure) { + consumer = CreateRawRetryingConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + } else { + consumer = CreateRawConsumer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + } + if (settings.Unpack) { + consumer = CreateDecompressingConsumer(std::move(consumer), settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + } + return consumer; +} + +std::shared_ptr<IConsumerImpl> TPQLibPrivate::CreateConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create consumer", "", ""); + std::shared_ptr<IConsumerImpl> consumer = CreateRawConsumer(settings, channelInfo, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + if (settings.Unpack) { + return CreateDecompressingConsumer(std::move(consumer), settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + } + return consumer; +} + +std::shared_ptr<TConsumer> TPQLibPrivate::CreateRawConsumerImpl(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create raw consumer", "", ""); + const bool addToDestroySet = refsAreDeadPtr != nullptr; + NThreading::TPromise<TConsumerCreateResponse> promise = NThreading::NewPromise<TConsumerCreateResponse>(); + std::shared_ptr<TConsumer> consumer = std::make_shared<TLocalConsumerImplCaller<TConsumer>>(settings, CQ, promise, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); + consumer->Init(); + if (addToDestroySet) { + AddToDestroySet(consumer); + } + return consumer; +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create raw retrying producer", settings.SourceId, ""); + const bool addToDestroySet = refsAreDeadPtr != nullptr; + auto producer = std::make_shared<TLocalProducerImplCaller<TRetryingProducer>>(settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); + producer->Init(); + if (addToDestroySet) { + AddToDestroySet(producer); + } + return std::move(producer); +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRetryingProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) +{ + auto newP = CreateNewProducer(settings, ChooseLogger(logger)); + if (newP) return newP; + + DEBUG_LOG("Create retrying producer", settings.SourceId, ""); + std::shared_ptr<IProducerImpl> rawProducer = CreateRawRetryingProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), settings.Codec, settings.Quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); + compressing->Init(); + if (!deprecated) { + AddToDestroySet(compressing); + } + return compressing; +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateRawMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create raw multicluster producer", settings.SourceIdPrefix, ""); + const bool addToDestroySet = refsAreDeadPtr != nullptr; + auto producer = std::make_shared<TLocalProducerImplCaller<TMultiClusterProducer>>(settings, std::move(refsAreDeadPtr), this, ChooseLogger(logger)); + producer->Init(); + if (addToDestroySet) { + AddToDestroySet(producer); + } + return std::move(producer); +} + +std::shared_ptr<IProducerImpl> TPQLibPrivate::CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create multicluster producer", settings.SourceIdPrefix, ""); + if (settings.ServerWeights.empty()) { + ythrow yexception() << "Can't create producer without server"; + } + const ECodec codec = settings.ServerWeights[0].ProducerSettings.Codec; + const int quality = settings.ServerWeights[0].ProducerSettings.Quality; + for (const auto& w : settings.ServerWeights) { + if (w.ProducerSettings.Codec != codec) { + ythrow yexception() << "Can't create multicluster producer with different codecs"; + } + } + + std::shared_ptr<IProducerImpl> rawProducer = CreateRawMultiClusterProducer(settings, GetSelfRefsAreDeadEvent(deprecated), ChooseLogger(logger)); + std::shared_ptr<IProducerImpl> compressing = std::make_shared<TLocalProducerImplCaller<TCompressingProducer>>(std::move(rawProducer), codec, quality, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); + compressing->Init(); + if (!deprecated) { + AddToDestroySet(compressing); + } + return compressing; +} + +std::shared_ptr<IProcessorImpl> TPQLibPrivate::CreateProcessor(const TProcessorSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger) +{ + DEBUG_LOG("Create processor", "", ""); + auto processor = std::make_shared<TLocalProcessorImplCaller<TProcessor>>(settings, GetSelfRefsAreDeadEvent(deprecated), this, ChooseLogger(logger)); + processor->Init(); + if (!deprecated) { + AddToDestroySet(processor); + } + return processor; +} + +TChannelHolder TPQLibPrivate::CreateChannel( + const TServerSetting& server, const TCredProviderPtr& credentialsProvider, TIntrusivePtr<ILogger> logger, + bool preferLocalProxy +) { + DEBUG_LOG("Create channel", "", ""); + TChannelHolder res; + res.ChannelPtr = new TChannel(server, credentialsProvider, this, ChooseLogger(logger), preferLocalProxy); + res.ChannelInfo = res.ChannelPtr->GetChannel(); + res.ChannelPtr->Start(); + return res; +} + + +TChannelHolder TPQLibPrivate::CreateChannel( + const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, bool preferLocalProxy +) { + DEBUG_LOG("Create channel", "", ""); + TChannelHolder res; + res.ChannelPtr = new TChannel(settings, this, ChooseLogger(logger), preferLocalProxy); + res.ChannelInfo = res.ChannelPtr->GetChannel(); + res.ChannelPtr->Start(); + return res; +} + +void TPQLibPrivate::CancelObjectsAndWait() { + { + auto guard = Guard(Lock); + DEBUG_LOG("Destroying PQLib. Cancelling objects: " << SyncDestroyedSet.size(), "", ""); + for (auto& ptr : SyncDestroyedSet) { + auto ref = ptr.lock(); + if (ref) { + ref->Cancel(); + } + } + SelfRefsAreDeadPtr = nullptr; + } + DEBUG_LOG("Destroying PQLib. Waiting refs to die", "", ""); + SelfRefsAreDeadEvent.Wait(); + + DEBUG_LOG("Destroying PQLib. HasDeprecatedObjects: " << AtomicGet(HasDeprecatedObjects) << ". Refs to PQLibPrivate: " << RefCount(), "", ""); + if (!AtomicGet(HasDeprecatedObjects)) { + Y_ASSERT(RefCount() == 1); + } +} + +void TPQLibPrivate::CreateSelfRefsAreDeadPtr() { + void* fakeAddress = &SelfRefsAreDeadEvent; + SelfRefsAreDeadPtr = std::shared_ptr<void>(fakeAddress, + [event = SelfRefsAreDeadEvent](void*) mutable { + event.Signal(); + }); +} + +void TPQLibPrivate::AddToDestroySet(std::weak_ptr<TSyncDestroyed> obj) { + auto guard = Guard(Lock); + // Check dead objects first + if (DestroySetAddCounter * 2 > SyncDestroyedSet.size()) { + for (auto i = SyncDestroyedSet.begin(); i != SyncDestroyedSet.end();) { + if (i->expired()) { + i = SyncDestroyedSet.erase(i); + } else { + ++i; + } + } + DestroySetAddCounter = 0; + } + ++DestroySetAddCounter; + SyncDestroyedSet.emplace(std::move(obj)); +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h new file mode 100644 index 0000000000..4c7145b048 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h @@ -0,0 +1,316 @@ +#pragma once + +#include "channel.h" +#include "internals.h" +#include "iconsumer_p.h" +#include "iprocessor_p.h" +#include "iproducer_p.h" +#include "scheduler.h" +#include "queue_pool.h" + +#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h> + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/iconsumer.h> + +#include <library/cpp/threading/future/future.h> + +#include <util/generic/ptr.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> +#include <util/generic/maybe.h> +#include <util/generic/queue.h> +#include <util/generic/guid.h> +#include <util/generic/hash_set.h> +#include <util/system/types.h> +#include <util/system/mutex.h> +#include <util/thread/factory.h> +#include <util/thread/pool.h> + +#include <util/datetime/base.h> + +#include <grpc++/create_channel.h> +#include <grpc++/completion_queue.h> +#include <google/protobuf/message.h> + +#include <atomic> +#include <type_traits> +#include <memory> +#include <set> +#include <thread> + +namespace NPersQueue { + +class TProducer; +class TConsumer; + + +struct TWrapperKey { + TServerSetting Settings; + std::shared_ptr<ICredentialsProvider> Provider; + + bool operator <(const TWrapperKey& b) const { + return Settings < b.Settings || (!(b.Settings < Settings) && Provider < b.Provider); + } + +}; + + + +class TCredentialsProviderWrapper : public NYdb::ICredentialsProvider { +public: + TCredentialsProviderWrapper(std::shared_ptr<NPersQueue::ICredentialsProvider> provider) + : Provider(provider) + {} + + bool IsValid() const override { + return true; + } + + TString GetAuthInfo() const override { + return GetToken(Provider.get()); + } + + private: + std::shared_ptr<NPersQueue::ICredentialsProvider> Provider; +}; + + +class TCredentialsProviderFactoryWrapper : public NYdb::ICredentialsProviderFactory { +public: + TString GetClientIdentity() const override { + return Guid; + } + + std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { + return Provider; + }; + + + TCredentialsProviderFactoryWrapper(std::shared_ptr<NPersQueue::ICredentialsProvider> provider) + : Provider(new TCredentialsProviderWrapper(provider)) + , Guid(CreateGuidAsString()) + {} + +private: + std::shared_ptr<NYdb::ICredentialsProvider> Provider; + TString Guid; +}; + +class TPQLibPrivate: public TAtomicRefCount<TPQLibPrivate> { + template <class T> + class TBaseFutureSubscriberObjectInQueue: public IObjectInQueue { + public: + void SetFuture(const NThreading::TFuture<T>& future) { + Future = future; + } + + protected: + NThreading::TFuture<T> Future; + }; + + template <class T, class TFunc> + class TFutureSubscriberObjectInQueue: public TBaseFutureSubscriberObjectInQueue<T> { + public: + template <class TF> + TFutureSubscriberObjectInQueue(TF&& f) + : Func(std::forward<TF>(f)) + { + } + + void Process(void*) override { + THolder<IObjectInQueue> holder(this); // like in TOwnedObjectInQueue::Process() + Func(this->Future); + } + + private: + TFunc Func; + }; + + template <class T, class TFunc> + static THolder<TBaseFutureSubscriberObjectInQueue<T>> MakeFutureSubscribeObjectInQueue(TFunc&& f) { + return MakeHolder<TFutureSubscriberObjectInQueue<T, std::remove_reference_t<TFunc>>>(std::forward<TFunc>(f)); + } + + template <class T> + class TSubscriberFunc { + public: + TSubscriberFunc(TPQLibPrivate* pqLib, const void* queueTag, THolder<TBaseFutureSubscriberObjectInQueue<T>> callback) + : PQLib(pqLib) + , QueueTag(queueTag) + , Callback(std::make_shared<THolder<TBaseFutureSubscriberObjectInQueue<T>>>(std::move(callback))) + { + } + + void operator()(const NThreading::TFuture<T>& future) { + if (Callback) { + (*Callback)->SetFuture(future); + THolder<IObjectInQueue> callback(std::move(*Callback)); + const bool added = PQLib->GetQueuePool().GetQueue(QueueTag).Add(callback.Get()); + if (added) { // can be false on shutdown + Y_UNUSED(callback.Release()); + } + } + } + + private: + TPQLibPrivate* PQLib; + const void* QueueTag; + // std::shared_ptr is only because copy constructor is required + std::shared_ptr<THolder<TBaseFutureSubscriberObjectInQueue<T>>> Callback; + }; + +public: + TPQLibPrivate(const TPQLibSettings& settings); + TPQLibPrivate(const TPQLibPrivate&) = delete; + TPQLibPrivate(TPQLibPrivate&&) = delete; + ~TPQLibPrivate(); + + TChannelHolder CreateChannel(const TServerSetting& server, const TCredProviderPtr& credentialsProvider, + TIntrusivePtr<ILogger> logger, bool preferLocalProxy = false); + TChannelHolder CreateChannel(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger, + bool preferLocalProxy = false); + + + std::shared_ptr<IProducerImpl> CreateRawProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IProducerImpl> CreateRawProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IProducerImpl> CreateProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IProducerImpl> CreateProducer(const TProducerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger); + + std::shared_ptr<IProducerImpl> CreateNewProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger); + + std::shared_ptr<IProducerImpl> CreateRawRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IProducerImpl> CreateRetryingProducer(const TProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); + + std::shared_ptr<IProducerImpl> CreateRawMultiClusterProducer(const TMultiClusterProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IProducerImpl> CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); + + std::shared_ptr<IConsumerImpl> CreateRawConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IConsumerImpl> CreateRawConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IConsumerImpl> CreateRawRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IConsumerImpl> CreateRawMultiClusterConsumer(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + + std::shared_ptr<IConsumerImpl> CreateConsumer(const TConsumerSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IConsumerImpl> CreateConsumer(const TConsumerSettings& settings, const TChannelInfo& channelInfo, bool deprecated, TIntrusivePtr<ILogger> logger); + + std::shared_ptr<IConsumerImpl> CreateNewConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger); + + std::shared_ptr<IProcessorImpl> CreateProcessor(const TProcessorSettings& settings, bool deprecated, TIntrusivePtr<ILogger> logger); + + TString GetUserAgent() const { + auto guard = Guard(UALock); + return UserAgent; + } + + void SetUserAgent(const TString& userAgent) { + auto guard = Guard(UALock); + UserAgent = userAgent; + } + + void Stop() { + ShuttingDown->store(true); + } + + const std::shared_ptr<grpc::CompletionQueue>& GetCompletionQueue() const { + return CQ; + } + + const TPQLibSettings& GetSettings() const { + return Settings; + } + + TScheduler& GetScheduler() { + return *Scheduler; + } + + TQueuePool& GetQueuePool() { + return QueuePool; + } + + IThreadPool& GetCompressionPool() { + return *CompressionPool; + } + + template <class TFunc, class T> + void Subscribe(const NThreading::TFuture<T>& future, const void* queueTag, TFunc&& func) { + TSubscriberFunc<T> f(this, queueTag, MakeFutureSubscribeObjectInQueue<T>(std::forward<TFunc>(func))); + future.Subscribe(std::move(f)); + } + + void SetLogger(TIntrusivePtr<ILogger> logger) { + Logger = std::move(logger); + } + + TIntrusivePtr<ILogger> ChooseLogger(TIntrusivePtr<ILogger> logger) { + return logger ? std::move(logger) : Logger; + } + + void CancelObjectsAndWait(); + + // for testing + std::shared_ptr<void> GetSelfRefsAreDeadPtr() const { + return SelfRefsAreDeadPtr; + } + + void AddToDestroySet(std::weak_ptr<TSyncDestroyed> obj); + +private: + template<typename TRawProducerImpl> + std::shared_ptr<TRawProducerImpl> CreateRawProducerImpl(const TProducerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<TConsumer> CreateRawConsumerImpl(const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + std::shared_ptr<IConsumerImpl> CreateDecompressingConsumer(std::shared_ptr<IConsumerImpl> subconsumer, const TConsumerSettings& settings, std::shared_ptr<void> refsAreDeadPtr, TIntrusivePtr<ILogger> logger); + + void CreateWrapper(const TWrapperKey& key, TIntrusivePtr<ILogger> logger); + + template <class TConsumerOrProducerSettings> + NYdb::NPersQueue::TPersQueueClient& GetOrCreatePersQueueClient(const TConsumerOrProducerSettings& settings, const TIntrusivePtr<ILogger>& logger); + + std::shared_ptr<void> GetSelfRefsAreDeadEvent(bool isObjectDeprecated) { + if (isObjectDeprecated) { + AtomicSet(HasDeprecatedObjects, 1); + } + return isObjectDeprecated ? nullptr : SelfRefsAreDeadPtr; + } + + void CreateSelfRefsAreDeadPtr(); + + using IThreadRef = TAutoPtr<IThreadFactory::IThread>; + +private: + const TPQLibSettings Settings; + + std::shared_ptr<grpc::CompletionQueue> CQ; + std::vector<std::thread> GRpcThreads; + + std::shared_ptr<std::atomic<bool>> ShuttingDown = std::make_shared<std::atomic<bool>>(false); // shared_ptr is for deprecated case when we have leaked threads + + THolder<TScheduler> Scheduler; + TQueuePool QueuePool; + std::shared_ptr<IThreadPool> CompressionPool; + + TIntrusivePtr<ILogger> Logger; + + TAtomic HasDeprecatedObjects = 0; + + TAutoEvent SelfRefsAreDeadEvent; + std::shared_ptr<void> SelfRefsAreDeadPtr; + TAdaptiveLock Lock, UALock; + size_t DestroySetAddCounter = 0; + std::set<std::weak_ptr<TSyncDestroyed>, std::owner_less<std::weak_ptr<TSyncDestroyed>>> SyncDestroyedSet; + + TString UserAgent = "C++ pqlib v0.3"; + + struct TYdbWrapper { + THolder<NYdb::NPersQueue::TPersQueueClient> PersQueueClient; + }; + + THolder<NYdb::TDriver> YdbDriver; + std::map<TWrapperKey, TYdbWrapper> Wrappers; + NYdb::NPersQueue::IExecutor::TPtr CompressExecutor; + NYdb::NPersQueue::IExecutor::TPtr HandlersExecutor; + std::shared_ptr<NYdb::ICredentialsProviderFactory> CredentialsProviderFactoryWrapper; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp new file mode 100644 index 0000000000..a695c2a8da --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p_ut.cpp @@ -0,0 +1,8 @@ +#include "persqueue_p.h" +#include "producer.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <library/cpp/testing/unittest/registar.h> + +namespace NPersQueue { +} +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp new file mode 100644 index 0000000000..1b3ca57fd8 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.cpp @@ -0,0 +1,358 @@ +#include "processor.h" +#include <util/string/builder.h> + +namespace NPersQueue { + +template<typename T> +bool IsReady(const NThreading::TFuture<T>& future) +{ + return future.HasValue() || future.HasException(); +} + +template<typename T> +void Reset(NThreading::TFuture<T>& future) +{ + future = NThreading::TFuture<T>(); +} + +template<typename T> +void TProcessor::SafeSubscribe(NThreading::TFuture<T>& future) +{ + std::weak_ptr<TProcessor> self(shared_from_this()); + PQLib->Subscribe( + future, + this, + [self](const auto&) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->ProcessFutures(); + }}); +} + +TProcessor::TProcessor(const TProcessorSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept + : IProcessorImpl(std::move(destroyEventRef), std::move(pqLib)) + , Settings(settings) + , Logger(std::move(logger)) +{} + +TProcessor::~TProcessor() noexcept +{ + SubscribeDestroyed(); + CleanState(); +} + +void TProcessor::Init() +{ + auto selfShared = shared_from_this(); + auto init = [selfShared] { + selfShared->InitImpl(); + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(this).AddFunc(init)); +} + +void TProcessor::InitImpl() +{ + ScheduleIdleSessionsCleanup(); + RecreateConsumer(); + SafeSubscribe(ConsumerIsStarted); + SafeSubscribe(ConsumerIsDead); +} + +// Must be called under lock +void TProcessor::CleanState() noexcept +{ + ProcessingData.clear(); + Reset(ConsumerIsDead); + Reset(ConsumerRequest); + Consumer.reset(); + Producers.clear(); + Data.clear(); + CurrentOriginDataMemoryUsage = 0; + CurrentProcessedDataMemoryUsage = 0; +} + + +bool TProcessor::NextRequest() noexcept +{ + Y_VERIFY(Consumer); + Y_VERIFY(!ConsumerRequest.Initialized()); + + DEBUG_LOG("Current memory usage: " + << "origin messages " << CurrentOriginDataMemoryUsage << "(" << Settings.MaxOriginDataMemoryUsage << "), " + << "processed messages " << CurrentProcessedDataMemoryUsage << "(" << Settings.MaxProcessedDataMemoryUsage << ")\n", + "", ""); + + if (CurrentOriginDataMemoryUsage < Settings.MaxOriginDataMemoryUsage && CurrentProcessedDataMemoryUsage < Settings.MaxProcessedDataMemoryUsage) { + DEBUG_LOG("Issuing consumer request\n", "", ""); + ConsumerRequest = Consumer->GetNextMessage(); + return true; + } else { + return false; + } +} + +bool TProcessor::ProcessConsumerResponse(TOriginData& result, TVector<NThreading::TFuture<TProcessedData>>& processedDataFutures) noexcept +{ + Y_VERIFY(ConsumerRequest.HasValue()); + auto message = ConsumerRequest.ExtractValueSync(); + Reset(ConsumerRequest); + const auto& type = message.Type; + const auto& resp = message.Response; + if (type == EMT_ERROR) { + // Will be handled later via ConsumerIsDead + ERR_LOG("Got error: " << resp << "\n", "", ""); + } else if (type == EMT_RELEASE) { + Y_VERIFY(resp.HasRelease()); + auto t = resp.GetRelease().GetTopic(); + auto p = resp.GetRelease().GetPartition(); + auto g = resp.GetRelease().GetGeneration(); + DEBUG_LOG("Got release for " << t << ":" << p << ", generation " << g << "\n", "", ""); + } else if (type == EMT_LOCK) { + Y_VERIFY(resp.HasLock()); + auto t = resp.GetLock().GetTopic(); + auto p = resp.GetLock().GetPartition(); + auto g = resp.GetLock().GetGeneration(); + DEBUG_LOG("Got lock for " << t << ":" << p << ", generation " << g << "\n", "", ""); + message.ReadyToRead.SetValue(TLockInfo{}); + } else if (type == EMT_DATA) { + Y_VERIFY(resp.HasData()); + Data.push_back(TDataInfo{resp.GetData().GetCookie()}); + for (auto& batch: resp.GetData().GetMessageBatch()) { + TPartition partition(batch.GetTopic(), batch.GetPartition()); + TVector<TOriginMessage>& originMessages = result.Messages[partition]; + TDeque<TProcessingData>& processingData = ProcessingData[partition]; + Data.back().TotalOriginMessages += batch.MessageSize(); + + for (auto& msg: batch.GetMessage()) { + CurrentOriginDataMemoryUsage += msg.ByteSizeLong(); + NThreading::TPromise<TProcessedData> promise = NThreading::NewPromise<TProcessedData>(); + NThreading::TFuture<TProcessedData> future = promise.GetFuture(); + TOriginMessage originMessage = TOriginMessage{msg, promise}; + processedDataFutures.push_back(future); + originMessages.push_back(originMessage); + processingData.push_back(TProcessingData{future, msg.GetOffset(), msg.ByteSizeLong(), &Data.back()}); + } + } + DEBUG_LOG("Processing data message " << resp.GetData().GetCookie() << "\n", "", ""); + return true; + } else if (type == EMT_COMMIT) { + DEBUG_LOG("Got commit", "", ""); + } else { + WARN_LOG("Unknown message type " << int(type) << ", response " << resp << "\n", "", ""); + } + + return false; +} + +// Must be called under lock +void TProcessor::RecreateConsumer() noexcept +{ + if (DestroyedFlag) { + return; + } + + INFO_LOG("Create consumer\n", "", ""); + CleanState(); + + TConsumerSettings consumerSettings = Settings.ConsumerSettings; + consumerSettings.UseLockSession = true; + + Consumer = PQLib->CreateConsumer(consumerSettings, DestroyEventRef == nullptr, Logger); + ConsumerIsStarted = Consumer->Start(); + ConsumerIsDead = Consumer->IsDead(); +} + +void TProcessor::ScheduleIdleSessionsCleanup() noexcept +{ + if (DestroyedFlag) { + return; + } + + std::weak_ptr<TProcessor> self(shared_from_this()); + PQLib->GetScheduler().Schedule(Settings.SourceIdIdleTimeout, + this, + [self] { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->CloseIdleSessions(); + } + }); +} + +void TProcessor::CloseIdleSessions() noexcept +{ + if (DestroyedFlag) { + return; + } + + TInstant now = TInstant::Now(); + for (auto it = Producers.begin(); it != Producers.end();) { + if (it->second.LastWriteTime + Settings.SourceIdIdleTimeout < now) { + INFO_LOG("Close producer for sourceid=" << it->first.SourceId, "", ""); + it = Producers.erase(it); + } else { + ++it; + } + } + ScheduleIdleSessionsCleanup(); +} + +void TProcessor::ProcessFutures() noexcept +{ + if (DestroyedFlag) { + return; + } + + TMaybe<NThreading::TFuture<TConsumerMessage>> consumerRequestFuture; + TVector<NThreading::TFuture<TProcessedData>> processedDataFutures; + TVector<NThreading::TFuture<TProducerCommitResponse>> producerAckFutures; + + if (IsReady(ConsumerIsDead)) { + ERR_LOG("Consumer died with error" << ConsumerIsDead.ExtractValueSync(), "", ""); + RecreateConsumer(); + SafeSubscribe(ConsumerIsStarted); + SafeSubscribe(ConsumerIsDead); + return; + } + + if (IsReady(ConsumerRequest)) { + TOriginData data; + bool hasData = ProcessConsumerResponse(data, processedDataFutures); + if (hasData) { + Requests.front().SetValue(data); + Requests.pop_front(); + } + } + + TInstant now = TInstant::Now(); + for (auto& partitionData: ProcessingData) { + TDeque<TProcessingData>& processingData = partitionData.second; + + while (!processingData.empty() && IsReady(processingData.front().Processed)) { + Y_VERIFY(processingData.front().Processed.HasValue(), "processing future cannot be filled with exception"); + ui64 originMessageOffset = processingData.front().OriginMessageOffset; + CurrentOriginDataMemoryUsage -= processingData.front().OriginMessageSize; + TProcessedData processedData = processingData.front().Processed.ExtractValueSync(); + TDataInfo* dataInfo = processingData.front().DataInfo; + dataInfo->OriginMessagesCounter++; + dataInfo->TotalProcessedMessages += processedData.Messages.size(); + processingData.pop_front(); + for (auto& processedMessage: processedData.Messages) { + TString newSourceId = processedMessage.SourceIdPrefix + partitionData.first.Topic + ":" + ToString(partitionData.first.PartitionId) + "_" + processedMessage.Topic; + TProducerKey producerKey { processedMessage.Topic, newSourceId }; + IProducer* producer = nullptr; + + auto it = Producers.find(producerKey); + if (it == Producers.end()) { + TProducerSettings producerSettings = Settings.ProducerSettings; + producerSettings.Topic = processedMessage.Topic; + producerSettings.SourceId = newSourceId; + producerSettings.PartitionGroup = processedMessage.Group; + producerSettings.ReconnectOnFailure = true; + Producers[producerKey] = TProducerInfo { PQLib->CreateRetryingProducer(producerSettings, DestroyEventRef == nullptr, Logger), {}, now}; + producer = Producers[producerKey].Producer.get(); + producer->Start(); + } else { + producer = it->second.Producer.get(); + } + + CurrentProcessedDataMemoryUsage += processedMessage.Data.size(); + DEBUG_LOG("Write message with seqNo=" << originMessageOffset + 1 << " to sourceId=" << newSourceId, "", ""); + NThreading::TFuture<TProducerCommitResponse> ack = producer->Write(originMessageOffset + 1, std::move(processedMessage.Data)); + producerAckFutures.push_back(ack); + Producers[producerKey].Queue.push_back(TProduceInfo(ack, dataInfo)); + } + } + } + + for (auto& producer: Producers) { + TProducerInfo& producerInfo = producer.second; + while (!producerInfo.Queue.empty() && IsReady(producerInfo.Queue.front().Ack)) { + auto ack = producerInfo.Queue.front().Ack.ExtractValueSync(); + DEBUG_LOG("Got ack for message with seqNo=" << ack.SeqNo << " from sourceId=" << producer.first.SourceId << " " << ack.Response, "", ""); + producerInfo.LastWriteTime = now; + producerInfo.Queue.front().DataInfo->ProcessedMessagesCounter++; + CurrentProcessedDataMemoryUsage -= ack.Data.GetSourceData().size(); + producerInfo.Queue.pop_front(); + } + } + + TVector<ui64> toCommit; + while (!Data.empty() && Data.front().IsReadyForCommit()) { + toCommit.push_back(Data.front().Cookie); + Data.pop_front(); + } + if (!toCommit.empty()) { + Consumer->Commit(toCommit); + } + + if (IsReady(ConsumerIsStarted) && !ConsumerRequest.Initialized() && !Requests.empty()) { + if (NextRequest()) { + consumerRequestFuture = ConsumerRequest; + } + } + + if (consumerRequestFuture.Defined()) { + SafeSubscribe(*consumerRequestFuture.Get()); + } + + for (auto& future: processedDataFutures) { + SafeSubscribe(future); + } + + for (auto& future: producerAckFutures) { + SafeSubscribe(future); + } +} + +void TProcessor::GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept +{ + Y_VERIFY(!DestroyedFlag); + TMaybe<NThreading::TFuture<TConsumerMessage>> consumerRequestFuture; + Requests.push_back(promise); + if (IsReady(ConsumerIsStarted) && !ConsumerRequest.Initialized()) { + if (NextRequest()) { + consumerRequestFuture = ConsumerRequest; + } + } + + if (consumerRequestFuture.Defined()) { + SafeSubscribe(*consumerRequestFuture.Get()); + } +} + +void TProcessor::SubscribeDestroyed() { + NThreading::TPromise<void> promise = ObjectsDestroyed; + auto handler = [promise](const auto&) mutable { + promise.SetValue(); + }; + std::vector<NThreading::TFuture<void>> futures; + futures.reserve(Producers.size() + 2); + futures.push_back(DestroyedPromise.GetFuture()); + if (Consumer) { + futures.push_back(Consumer->Destroyed()); + } + for (const auto& p : Producers) { + if (p.second.Producer) { + futures.push_back(p.second.Producer->Destroyed()); + } + } + WaitExceptionOrAll(futures).Subscribe(handler); +} + +NThreading::TFuture<void> TProcessor::Destroyed() noexcept { + return ObjectsDestroyed.GetFuture(); +} + +void TProcessor::Cancel() { + DestroyedFlag = true; + + for (auto& req : Requests) { + req.SetValue(TOriginData()); + } + Requests.clear(); + + DestroyPQLibRef(); +} + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h new file mode 100644 index 0000000000..a6abb827a0 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor.h @@ -0,0 +1,118 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include "persqueue_p.h" + +#include <library/cpp/threading/future/future.h> +#include <util/generic/deque.h> +#include <util/generic/map.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +namespace NPersQueue { + +struct TDataInfo { + ui64 Cookie; + ui64 TotalOriginMessages; + ui64 OriginMessagesCounter; + ui64 TotalProcessedMessages; + ui64 ProcessedMessagesCounter; + + TDataInfo(ui64 cookie) + : Cookie(cookie) + , TotalOriginMessages(0) + , OriginMessagesCounter(0) + , TotalProcessedMessages(0) + , ProcessedMessagesCounter(0) + {} + + bool IsReadyForCommit() { + return TotalOriginMessages == OriginMessagesCounter && TotalProcessedMessages == ProcessedMessagesCounter; + } +}; + +struct TProducerKey { + TString Topic; + TString SourceId; + + bool operator <(const TProducerKey& other) const { + return std::tie(Topic, SourceId) < std::tie(other.Topic, other.SourceId); + } +}; + +struct TProduceInfo { + NThreading::TFuture<TProducerCommitResponse> Ack; + TDataInfo* DataInfo = nullptr; + + explicit TProduceInfo(const NThreading::TFuture<TProducerCommitResponse>& ack, TDataInfo* dataInfo) + : Ack(ack) + , DataInfo(dataInfo) + {} +}; + +struct TProducerInfo { + std::shared_ptr<IProducerImpl> Producer; + TDeque<TProduceInfo> Queue; + TInstant LastWriteTime; +}; + +struct TProcessingData { + NThreading::TFuture<TProcessedData> Processed; + ui64 OriginMessageOffset; + ui64 OriginMessageSize; + TDataInfo* DataInfo; + + TProcessingData(const NThreading::TFuture<TProcessedData>& processed, ui64 originMessageOffset, ui64 originMessageSize, TDataInfo* dataInfo) + : Processed(processed) + , OriginMessageOffset(originMessageOffset) + , OriginMessageSize(originMessageSize) + , DataInfo(dataInfo) + {}; +}; + +class TPQLibPrivate; + +class TProcessor: public IProcessorImpl, public std::enable_shared_from_this<TProcessor> { +public: + TProcessor(const TProcessorSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept; + ~TProcessor() noexcept; + void Init() override; + + using IProcessorImpl::GetNextData; + void GetNextData(NThreading::TPromise<TOriginData>& promise) noexcept override; + + NThreading::TFuture<void> Destroyed() noexcept override; + + void Cancel() override; + +protected: + TProcessorSettings Settings; + TIntrusivePtr<ILogger> Logger; + std::shared_ptr<IConsumerImpl> Consumer; + NThreading::TFuture<TConsumerCreateResponse> ConsumerIsStarted; + NThreading::TFuture<TError> ConsumerIsDead; + NThreading::TFuture<TConsumerMessage> ConsumerRequest; + TMap<TProducerKey, TProducerInfo> Producers; + ui64 CurrentOriginDataMemoryUsage; + ui64 CurrentProcessedDataMemoryUsage; + + TDeque<NThreading::TPromise<TOriginData>> Requests; + TDeque<TDataInfo> Data; + TMap<TPartition, TDeque<TProcessingData>> ProcessingData; + NThreading::TPromise<void> ObjectsDestroyed = NThreading::NewPromise<void>(); + bool DestroyedFlag = false; + + template<typename T> + void SafeSubscribe(NThreading::TFuture<T>& future); + void CleanState() noexcept; + bool NextRequest() noexcept; + void RecreateConsumer() noexcept; + void ScheduleIdleSessionsCleanup() noexcept; + void CloseIdleSessions() noexcept; + void ProcessFutures() noexcept; + bool ProcessConsumerResponse(TOriginData& result, TVector<NThreading::TFuture<TProcessedData>>& processedDataFutures) noexcept; + void SubscribeDestroyed(); + void InitImpl(); +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp new file mode 100644 index 0000000000..d36573dcfc --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/processor_ut.cpp @@ -0,0 +1,133 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + +namespace NPersQueue { +Y_UNIT_TEST_SUITE(TProcessorTest) { + TConsumerSettings MakeConsumerSettings(const TString& clientId, const TVector<TString>& topics, + const TTestServer& testServer) { + TConsumerSettings consumerSettings; + consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + consumerSettings.ClientId = clientId; + consumerSettings.Topics = topics; + consumerSettings.UseLockSession = false; + return consumerSettings; + } + + TProducerSettings MakeProducerSettings(const TString& topic, const TTestServer& testServer) { + TProducerSettings producerSettings; + producerSettings.ReconnectOnFailure = true; + producerSettings.Topic = topic; + producerSettings.SourceId = "123"; + producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + producerSettings.Codec = ECodec::RAW; + return producerSettings; + } + + TProcessorSettings MakeProcessorSettings(const TTestServer& testServer) { + TProcessorSettings processorSettings; + processorSettings.ConsumerSettings = MakeConsumerSettings("processor", {"input"}, testServer); + processorSettings.ProducerSettings = MakeProducerSettings("output", testServer); + return processorSettings; + } + + void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { + const TProducerCommitResponse& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); + } + + void ProcessMessage(TOriginData& originData) { + for (auto& messages: originData.Messages) { + for (auto& message: messages.second) { + auto msg = message.Message; + + TVector<TProcessedMessage> processedMessages; + for (int i = 0; i < 3; ++i) { + TStringBuilder sourceIdPrefix; + sourceIdPrefix << "shard" << i << "_"; + processedMessages.push_back(TProcessedMessage {"output", ToString(msg.GetOffset()), sourceIdPrefix, 0}); + } + message.Processed.SetValue(TProcessedData{processedMessages}); + } + } + } + + bool IsFinished(const TTestServer& testServer) { + auto clientInfo = testServer.AnnoyingClient->GetClientInfo({"rt3.dc1--input"}, "processor", true); + auto topicResult = clientInfo.GetMetaResponse().GetCmdGetReadSessionsInfoResult().GetTopicResult(0).GetPartitionResult(); + for (auto& partition: topicResult) { + if (partition.GetClientOffset() < partition.GetEndOffset()) return false; + } + return true; + } + + void BaseFunctionalityTest() { + TTestServer testServer(false); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--input", 2); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--output", 4); + + auto producerSettings = MakeProducerSettings("input", testServer); + auto producer = testServer.PQLib->CreateProducer(producerSettings, testServer.PQLibSettings.DefaultLogger, false); + producer->Start(); + + for (int i = 1; i < 4; ++i) { + auto write1 = producer->Write(i, TString("blob")); + AssertWriteValid(write1); + } + + auto processorSettings = MakeProcessorSettings(testServer); + auto processor = testServer.PQLib->CreateProcessor(processorSettings, testServer.PQLibSettings.DefaultLogger, false); + + while (!IsFinished(testServer)) { + auto data = processor->GetNextData(); + data.Subscribe([](const auto& f) { + auto value = f.GetValueSync(); + ProcessMessage(value); + }); + Sleep(TDuration::MilliSeconds(100)); + } + + auto consumerSettings = MakeConsumerSettings("checker", {"output"}, testServer); + auto consumer = testServer.PQLib->CreateConsumer(consumerSettings, nullptr, false); + auto isStarted = consumer->Start().ExtractValueSync(); + UNIT_ASSERT_C(isStarted.Response.HasInit(), "cannot start consumer " << isStarted.Response); + + ui32 totalMessages = 9; + ui32 currentMessages = 0; + THashMap<TString, ui32> expectedSeqno = { + {"shard0_rt3.dc1--input:0_output", 1}, {"shard1_rt3.dc1--input:0_output", 1}, {"shard2_rt3.dc1--input:0_output", 1}, + {"shard0_rt3.dc1--input:1_output", 1}, {"shard1_rt3.dc1--input:1_output", 1}, {"shard2_rt3.dc1--input:1_output", 1}, + {"shard0_rt3.dc1--input:2_output", 1}, {"shard1_rt3.dc1--input:2_output", 1}, {"shard2_rt3.dc1--input:2_output", 1}, + {"shard0_rt3.dc1--input:3_output", 1}, {"shard1_rt3.dc1--input:3_output", 1}, {"shard2_rt3.dc1--input:3_output", 1}, + }; + while (currentMessages < totalMessages) { + auto f = consumer->GetNextMessage(); + for (auto& batch: f.GetValueSync().Response.GetData().GetMessageBatch()) { + for (auto& message: batch.GetMessage()) { + TString sourceId = message.GetMeta().GetSourceId(); + ui32 seqNo = message.GetMeta().GetSeqNo(); + UNIT_ASSERT_EQUAL(ToString(seqNo - 1), message.GetData()); + auto it = expectedSeqno.find(sourceId); + UNIT_ASSERT_UNEQUAL_C(it, expectedSeqno.end(), "unknown sourceId " << sourceId); + UNIT_ASSERT_EQUAL(it->second, seqNo); + it->second++; + currentMessages += 1; + } + } + } + } + + Y_UNIT_TEST(BasicFunctionality) { + BaseFunctionalityTest(); + } +} +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp new file mode 100644 index 0000000000..2de983453f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.cpp @@ -0,0 +1,506 @@ +#include <util/generic/strbuf.h> +#include <util/stream/zlib.h> +#include <util/string/cast.h> +#include <util/string/printf.h> +#include <util/string/vector.h> +#include <util/string/builder.h> + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/persqueue_p.h> + +#include <grpc++/create_channel.h> + + +namespace NPersQueue { + +class TProducerDestroyHandler : public IHandler { +public: + TProducerDestroyHandler(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : Ptr(std::move(ptr)) + , RpcStuff(std::move(rpcStuff)) + , QueueTag(queueTag) + , PQLib(pqLib) + {} + + void Destroy(const TError&) override { + auto producer = Ptr; + auto handler = [producer] { + auto producerShared = producer.lock(); + if (producerShared) { + producerShared->OnFail(); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + + TString ToString() override { return "TProducerDestroyHandler"; } + +protected: + std::weak_ptr<TProducer> Ptr; + TIntrusivePtr<TProducer::TRpcStuff> RpcStuff; // must simply live + const void* QueueTag; + TPQLibPrivate* PQLib; +}; + +class TStreamCreated : public TProducerDestroyHandler { +public: + TStreamCreated(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) + {} + + void Done() override { + auto producer = Ptr; + auto UA = PQLib->GetUserAgent(); + auto handler = [producer, UA] { + auto producerShared = producer.lock(); + if (producerShared) { + producerShared->OnStreamCreated(UA); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + + TString ToString() override { return "StreamCreated"; } +}; + +class TFinishDone : public TProducerDestroyHandler { +public: + TFinishDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) + {} + + void Done() override { + auto producer = Ptr; + auto handler = [producer] { + auto producerShared = producer.lock(); + if (producerShared) { + producerShared->OnFinishDone(); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + + void Destroy(const TError&) override { + Y_FAIL("Finish call failed"); + } + + TString ToString() override { return "Finish"; } +}; + + +class TWriteDone : public TProducerDestroyHandler { +public: + TWriteDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) + {} + + void Done() override { + auto producer = Ptr; + auto handler = [producer] { + auto producerShared = producer.lock(); + if (producerShared) { + producerShared->OnWriteDone(); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + TString ToString() override { return "WriteDone"; } +}; + + +class TReadDone : public TProducerDestroyHandler { +public: + TReadDone(std::weak_ptr<TProducer> ptr, TIntrusivePtr<TProducer::TRpcStuff> rpcStuff, + const void* queueTag, TPQLibPrivate* pqLib) + : TProducerDestroyHandler(std::move(ptr), std::move(rpcStuff), queueTag, pqLib) + {} + + void Done() override { + auto producer = Ptr; + auto handler = [producer] { + auto producerShared = producer.lock(); + if (producerShared) { + producerShared->OnReadDone(); + } + }; + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(handler)); + } + TString ToString() override { return "ReadDone"; } +}; + +TProducer::TProducer(const TProducerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, + NThreading::TPromise<TProducerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept + : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) + , RpcStuff(new TRpcStuff()) + , Settings(settings) + , StartPromise(promise) + , IsDeadPromise(NThreading::NewPromise<TError>()) + , Pos(0) + , WriteInflight(false) + , ProxyCookie(0) + , Logger(std::move(logger)) + , IsDestroyed(false) + , IsDestroying(false) + , Failing(false) +{ + RpcStuff->CQ = std::move(cq); +} + +TProducer::~TProducer() { + Destroy(); +} + +void TProducer::Destroy() noexcept { + IsDestroying = true; + RpcStuff->Context.TryCancel(); + ChannelHolder.ChannelPtr = nullptr; + + Destroy("Destructor called"); +} + +void TProducer::DoStart(TInstant deadline) { + if (IsDestroyed) { + return; + } + + if (deadline != TInstant::Max()) { + std::weak_ptr<TProducer> self = shared_from_this(); + auto onStartTimeout = [self] { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnStartTimeout(); + } + }; + StartDeadlineCallback = + PQLib->GetScheduler().Schedule(deadline, this, onStartTimeout); + } + + FillMetaHeaders(RpcStuff->Context, Settings.Server.Database, Settings.CredentialsProvider.get()); + RpcStuff->Stub = PersQueueService::NewStub(RpcStuff->Channel); + RpcStuff->Stream = RpcStuff->Stub->AsyncWriteSession(&RpcStuff->Context, RpcStuff->CQ.get(), new TQueueEvent(StreamCreatedHandler)); +} + +void TProducer::Init() { + std::weak_ptr<TProducer> self = shared_from_this(); // we can't make this object in constructor, because this will be the only reference to us and ~shared_ptr() will destroy us. + StreamCreatedHandler.Reset(new TStreamCreated(self, RpcStuff, this, PQLib.Get())); + ReadDoneHandler.Reset(new TReadDone(self, RpcStuff, this, PQLib.Get())); + WriteDoneHandler.Reset(new TWriteDone(self, RpcStuff, this, PQLib.Get())); + FinishDoneHandler.Reset(new TFinishDone(self, RpcStuff, this, PQLib.Get())); +} + +NThreading::TFuture<TProducerCreateResponse> TProducer::Start(TInstant deadline) noexcept { + if (IsDestroyed) { + TWriteResponse res; + res.MutableError()->SetDescription("Producer is dead"); + res.MutableError()->SetCode(NErrorCode::ERROR); + return NThreading::MakeFuture<TProducerCreateResponse>(TProducerCreateResponse{std::move(res)}); + } + NThreading::TFuture<TProducerCreateResponse> ret = StartPromise.GetFuture(); + if (ChannelHolder.ChannelInfo.Initialized()) { + std::weak_ptr<TProducer> self = shared_from_this(); + PQLib->Subscribe(ChannelHolder.ChannelInfo, + this, + [self, deadline](const auto& f) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->SetChannel(f.GetValue()); + selfShared->DoStart(deadline); + } + }); + } else { + if (RpcStuff->Channel && ProxyCookie) { + DoStart(deadline); + } else { + Destroy("No channel"); + } + } + return ret; +} + +void TProducer::SetChannel(const TChannelHolder& channel) noexcept { + if (IsDestroyed) { + return; + } + + ChannelHolder = channel; +} + +void TProducer::SetChannel(const TChannelInfo& channel) noexcept { + if (IsDestroyed) { + return; + } + + Y_VERIFY(!RpcStuff->Channel); + if (!channel.Channel || channel.ProxyCookie == 0) { + if (channel.Channel) { + Destroy("ProxyCookie is zero"); + } else { + Destroy("Channel creation error"); + } + + return; + } + + RpcStuff->Channel = channel.Channel; + ProxyCookie = channel.ProxyCookie; +} + + +static TProducerCommitResponse MakeErrorCommitResponse(const TString& description, TProducerSeqNo seqNo, const TData& data) { + TWriteResponse res; + res.MutableError()->SetDescription(description); + res.MutableError()->SetCode(NErrorCode::ERROR); + return TProducerCommitResponse{seqNo, data, std::move(res)}; +} + +void TProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept { + return TProducer::Write(promise, MaxSeqNo + 1, std::move(data)); +} + +void TProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept { + Y_VERIFY(data.IsEncoded()); + + MaxSeqNo = Max(seqNo, MaxSeqNo); + + if (IsDestroyed) { + promise.SetValue(MakeErrorCommitResponse("producer is dead", seqNo, data)); + return; + } + + if (StartPromise.Initialized()) { + promise.SetValue(MakeErrorCommitResponse("producer is not ready", seqNo, data)); + return; + } + + CommitPromises.push_back(promise); + Data.emplace_back(seqNo, std::move(data)); + DoWrite(); +} + +void TProducer::OnWriteDone() { + if (IsDestroyed) { + return; + } + + WriteInflight = false; + DoWrite(); +} + +void TProducer::DoWrite() { + if (Failing && !WriteInflight) { + return; + } + if (WriteInflight || Pos == Data.size()) { + return; + } + WriteInflight = true; + TWriteRequest req; + Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials()); + + req.MutableData()->SetSeqNo(Data[Pos].SeqNo); + req.MutableData()->SetData(Data[Pos].Data.GetEncodedData()); + req.MutableData()->SetCodec(Data[Pos].Data.GetCodecType()); + req.MutableData()->SetCreateTimeMs(Data[Pos].Data.GetTimestamp().MilliSeconds()); + + ++Pos; + RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); +} + +NThreading::TFuture<TError> TProducer::IsDead() noexcept { + return IsDeadPromise.GetFuture(); +} + +void TProducer::Destroy(const TString& description) { + TError error; + error.SetDescription(description); + error.SetCode(NErrorCode::ERROR); + Destroy(error); +} + +void TProducer::Destroy(const TError& error) { + if (IsDestroyed) { + return; + } + + if (!IsDestroying) { + INFO_LOG("Error: " << error, Settings.SourceId, SessionId); + } + + IsDestroyed = true; + IsDeadPromise.SetValue(error); + + if (StartDeadlineCallback) { + StartDeadlineCallback->TryCancel(); + } + + NThreading::TFuture<TChannelInfo> tmp; + tmp.Swap(ChannelHolder.ChannelInfo); + ChannelHolder.ChannelPtr = nullptr; + + Error = error; + if (StartPromise.Initialized()) { + NThreading::TPromise<TProducerCreateResponse> tmp; + tmp.Swap(StartPromise); + TWriteResponse res; + res.MutableError()->CopyFrom(Error); + tmp.SetValue(TProducerCreateResponse{std::move(res)}); + } + while (!CommitPromises.empty()) { + auto p = CommitPromises.front(); + CommitPromises.pop_front(); + auto pp(std::move(Data.front())); + Data.pop_front(); + if (Pos > 0) { + --Pos; + } + TWriteResponse res; + res.MutableError()->CopyFrom(Error); + + p.SetValue(TProducerCommitResponse{pp.SeqNo, std::move(pp.Data), std::move(res)}); + } + + StreamCreatedHandler.Reset(); + ReadDoneHandler.Reset(); + WriteDoneHandler.Reset(); + FinishDoneHandler.Reset(); + + DestroyPQLibRef(); +} + +void TProducer::OnStreamCreated(const TString& userAgent) { + if (IsDestroyed) { + return; + } + + TWriteRequest req; + Settings.CredentialsProvider->FillAuthInfo(req.MutableCredentials()); + + req.MutableInit()->SetSourceId(Settings.SourceId); + req.MutableInit()->SetTopic(Settings.Topic); + req.MutableInit()->SetProxyCookie(ProxyCookie); + req.MutableInit()->SetPartitionGroup(Settings.PartitionGroup); + req.MutableInit()->SetVersion(userAgent); + Y_VERIFY(!userAgent.empty()); + + for (const auto& attr : Settings.ExtraAttrs) { + auto item = req.MutableInit()->MutableExtraFields()->AddItems(); + item->SetKey(attr.first); + item->SetValue(attr.second); + } + + WriteInflight = true; + RpcStuff->Stream->Write(req, new TQueueEvent(WriteDoneHandler)); + + RpcStuff->Response.Clear(); + RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); +} + +void TProducer::OnFail() { + if (Failing) + return; + Failing = true; + + if (IsDestroyed) { + return; + } + + RpcStuff->Stream->Finish(&RpcStuff->Status, new TQueueEvent(FinishDoneHandler)); +} + +void TProducer::OnFinishDone() { + if (IsDestroyed) { + return; + } + + TError error; + const auto& msg = RpcStuff->Status.error_message(); + TString reason(msg.data(), msg.length()); + error.SetDescription(reason); + error.SetCode(NErrorCode::ERROR); + + Destroy(error); +} + +void TProducer::OnReadDone() { + if (IsDestroyed) { + return; + } + + if (RpcStuff->Response.HasError()) { + Destroy(RpcStuff->Response.GetError()); + return; + } + + if (StartPromise.Initialized()) { //init response + NThreading::TPromise<TProducerCreateResponse> tmp; + tmp.Swap(StartPromise); + Y_VERIFY(RpcStuff->Response.HasInit()); + auto res(std::move(RpcStuff->Response)); + RpcStuff->Response.Clear(); + + SessionId = res.GetInit().GetSessionId(); + const ui32 partition = res.GetInit().GetPartition(); + MaxSeqNo = res.GetInit().GetMaxSeqNo(); + const TProducerSeqNo seqNo = MaxSeqNo; + const TString topic = res.GetInit().GetTopic(); + tmp.SetValue(TProducerCreateResponse{std::move(res)}); + + if (StartDeadlineCallback) { + StartDeadlineCallback->TryCancel(); + } + StartDeadlineCallback = nullptr; + + RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); + + DEBUG_LOG("Stream created to topic " << topic << " partition " << partition << " maxSeqNo " << seqNo, Settings.SourceId, SessionId); + } else { //write response + //get first CommitPromise + Y_VERIFY(!Data.empty()); + auto p = CommitPromises.front(); + CommitPromises.pop_front(); + auto pp(std::move(Data.front())); + Data.pop_front(); + Y_VERIFY(Pos > 0); + --Pos; + + Y_VERIFY(RpcStuff->Response.HasAck()); + Y_VERIFY(RpcStuff->Response.GetAck().GetSeqNo() == pp.SeqNo); + auto res(std::move(RpcStuff->Response)); + RpcStuff->Response.Clear(); + RpcStuff->Stream->Read(&RpcStuff->Response, new TQueueEvent(ReadDoneHandler)); + + p.SetValue(TProducerCommitResponse{pp.SeqNo, std::move(pp.Data), std::move(res)}); + } +} + +void TProducer::OnStartTimeout() { + if (IsDestroyed) { + return; + } + + StartDeadlineCallback = nullptr; + if (!StartPromise.Initialized()) { + // everything is OK, there is no timeout, we have already started. + return; + } + + TError error; + error.SetDescription("Start timeout"); + error.SetCode(NErrorCode::CREATE_TIMEOUT); + Destroy(error); +} + +void TProducer::Cancel() { + IsDestroying = true; + RpcStuff->Context.TryCancel(); + ChannelHolder.ChannelPtr = nullptr; + + Destroy(GetCancelReason()); +} + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h new file mode 100644 index 0000000000..af74ced4b9 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer.h @@ -0,0 +1,107 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/channel.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/iproducer_p.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/internals.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h> +#include <library/cpp/threading/future/future.h> +#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> +#include <deque> + +namespace NPersQueue { + +class TProducer : public IProducerImpl, public std::enable_shared_from_this<TProducer> { +public: + using IProducerImpl::Write; + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept override; + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override; + + NThreading::TFuture<TError> IsDead() noexcept override; + + NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; + + TProducer(const TProducerSettings& settings, std::shared_ptr<grpc::CompletionQueue> cq, + NThreading::TPromise<TProducerCreateResponse> promise, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) noexcept; + ~TProducer(); + + void Destroy() noexcept; + void SetChannel(const TChannelHolder& channel) noexcept; + void SetChannel(const TChannelInfo& channel) noexcept; + + void DoStart(TInstant deadline); + + void Init() override; + + const TProducerSettings& GetSettings() const { + return Settings; + } + + void Cancel() override; + +public: + using TStream = grpc::ClientAsyncReaderWriterInterface<TWriteRequest, TWriteResponse>; + + // objects that must live after destruction of producer untill all the callbacks arrive at CompletionQueue + struct TRpcStuff: public TAtomicRefCount<TRpcStuff> { + TWriteResponse Response; + std::shared_ptr<grpc::CompletionQueue> CQ; + std::shared_ptr<grpc::Channel> Channel; + std::unique_ptr<PersQueueService::Stub> Stub; + grpc::ClientContext Context; + std::unique_ptr<TStream> Stream; + grpc::Status Status; + }; + +private: + friend class TPQLibPrivate; + friend class TStreamCreated; + friend class TReadDone; + friend class TWriteDone; + friend class TProducerDestroyHandler; + friend class TFinishDone; + + IHandlerPtr StreamCreatedHandler; + IHandlerPtr ReadDoneHandler; + IHandlerPtr WriteDoneHandler; + IHandlerPtr FinishDoneHandler; + + void Destroy(const TError& error); + void Destroy(const TString& description); // the same but with Code=ERROR + void OnStreamCreated(const TString& userAgent); + void OnReadDone(); + void OnWriteDone(); + void DoWrite(); + void OnFail(); + void OnFinishDone(); + void OnStartTimeout(); + +protected: + TIntrusivePtr<TRpcStuff> RpcStuff; + + TChannelHolder ChannelHolder; + TProducerSettings Settings; + + TString SessionId; + + NThreading::TPromise<TProducerCreateResponse> StartPromise; + NThreading::TPromise<TError> IsDeadPromise; + std::deque<NThreading::TPromise<TProducerCommitResponse>> CommitPromises; + std::deque<TWriteData> Data; + ui32 Pos = 0; + bool WriteInflight; + ui64 ProxyCookie = 0; + TProducerSeqNo MaxSeqNo = 0; + + TIntrusivePtr<ILogger> Logger; + + TError Error; + + bool IsDestroyed; + bool IsDestroying; + bool Failing; + TIntrusivePtr<TScheduler::TCallbackHandler> StartDeadlineCallback; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp new file mode 100644 index 0000000000..5f3deb880d --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/producer_ut.cpp @@ -0,0 +1,202 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> +#include <library/cpp/testing/unittest/registar.h> + +namespace NPersQueue { + +Y_UNIT_TEST_SUITE(TProducerTest) { + Y_UNIT_TEST(NotStartedProducerCanBeDestructed) { + // Test that producer doesn't hang on till shutdown + TPQLib lib; + TProducerSettings settings; + settings.Server = TServerSetting{"localhost"}; + settings.Topic = "topic"; + settings.SourceId = "src"; + lib.CreateProducer(settings, {}, false); + } + + TProducerSettings MakeProducerSettings(const TTestServer& testServer) { + TProducerSettings producerSettings; + producerSettings.ReconnectOnFailure = false; + producerSettings.Topic = "topic1"; + producerSettings.SourceId = "123"; + producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + producerSettings.Codec = ECodec::LZOP; + return producerSettings; + } + + Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + const size_t partitions = 1; + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + + testServer.WaitInit("topic1"); + + auto producer = testServer.PQLib->CreateProducer(MakeProducerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false); + UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); + auto isDead = producer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + auto write1 = producer->Write(1, TString("blob1")); + auto write2 = producer->Write(2, TString("blob2")); + auto write3 = producer->Write(3, TString("blob3")); + auto write4 = producer->Write(4, TString("blob4")); + auto write5 = producer->Write(5, TString("blob5")); + + testServer.PQLib = nullptr; + Cerr << "PQLib destroyed" << Endl; + + UNIT_ASSERT(write1.HasValue()); + UNIT_ASSERT(write2.HasValue()); + UNIT_ASSERT(write3.HasValue()); + UNIT_ASSERT(write4.HasValue()); + UNIT_ASSERT(write5.HasValue()); + + UNIT_ASSERT(write1.GetValue().Response.HasError()); + UNIT_ASSERT(write2.GetValue().Response.HasError()); + UNIT_ASSERT(write3.GetValue().Response.HasError()); + UNIT_ASSERT(write4.GetValue().Response.HasError()); + UNIT_ASSERT(write5.GetValue().Response.HasError()); + + auto write6 = producer->Write(6, TString("blob6")); + UNIT_ASSERT(write6.HasValue()); + UNIT_ASSERT(write6.GetValue().Response.HasError()); + } + + Y_UNIT_TEST(WriteToDeadProducer) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLibSettings pqLibSettings; + pqLibSettings.DefaultLogger = logger; + THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); + + auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, false); + auto f = producer->Start(); + UNIT_ASSERT(f.GetValueSync().Response.HasError()); + Cerr << f.GetValueSync().Response << "\n"; + auto isDead = producer->IsDead(); + isDead.Wait(); + UNIT_ASSERT(isDead.HasValue()); + + auto write = producer->Write(1, TString("blob")); + + Cerr << write.GetValueSync().Response << "\n"; + UNIT_ASSERT(write.GetValueSync().Response.HasError()); + UNIT_ASSERT_STRINGS_EQUAL(write.GetValueSync().Response.GetError().GetDescription(), "Destroyed"); + } + + Y_UNIT_TEST(Auth_StartProducerWithInvalidTokenFromGrpcMetadataPointOfView_StartFailedAndProduerDiedWithBadRequestErrors) { + TPQLibSettings pqLibSettings; + if (!std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED")) + return; + + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(); + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + pqLibSettings.DefaultLogger = logger; + THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); + + auto fillCredentialsWithInvalidToken = [](NPersQueue::TCredentials* authInfo) { + authInfo->set_tvm_service_ticket("token\n"); + }; + auto producerSettings = MakeProducerSettings(testServer); + producerSettings.CredentialsProvider = std::make_shared<TCallbackCredentialsProvider>(std::move(fillCredentialsWithInvalidToken)); + auto producer = PQLib->CreateProducer(producerSettings, logger, false); + + + Cerr << "Wait for producer start procedure end" << Endl; + auto startResponse = producer->Start().GetValueSync().Response; + + UNIT_ASSERT(startResponse.has_error()); + UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::ERROR, startResponse.error().code(), startResponse); + + Cerr << "Wait for producer death" << Endl; + auto deathCause = producer->IsDead().GetValueSync(); + + + UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::ERROR, deathCause.code(), deathCause); + } + + Y_UNIT_TEST(Codecs_WriteWithNonDefaultCodecThatRequiresAdditionalConfiguration_ConsumerDiesWithBadRequestError) { + SDKTestSetup setup{"Codecs_WriteWithNonDefaultCodecThatRequiresAdditionalConfiguration_ConsumerIsDeadWithBadRequestError"}; + auto log = setup.GetLog(); + auto producerSettings = setup.GetProducerSettings(); + // TTestServer::AnnoyingClient creates topic with default codecs set: raw, gzip, lzop. zstd not included + producerSettings.Codec = ECodec::ZSTD; + auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); + log << TLOG_INFO << "Wait for producer start"; + auto startResponse = producer->Start().GetValueSync().Response; + UNIT_ASSERT_C(!startResponse.HasError(), startResponse); + + log << TLOG_INFO << "Wait for write response"; + auto writeResponse = producer->Write(NUnitTest::RandomString(250 * 1024, std::rand())).GetValueSync().Response; + + + UNIT_ASSERT(writeResponse.HasError()); + Cerr << writeResponse << "\n"; + UNIT_ASSERT_EQUAL_C(NPersQueue::NErrorCode::BAD_REQUEST, writeResponse.GetError().GetCode(), writeResponse); + } + + void StartProducerWithDiscovery(bool cds, const TString& preferredCluster = {}, bool addBrokenDatacenter = !GrpcV1EnabledByDefault()) { + SDKTestSetup setup{"StartProducerWithDiscovery", false}; + setup.Start(true, addBrokenDatacenter); + auto log = setup.GetLog(); + auto producerSettings = setup.GetProducerSettings(); + if (preferredCluster) { + producerSettings.PreferredCluster = preferredCluster; + } + auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); + auto startResponse = producer->Start().GetValueSync().Response; + UNIT_ASSERT_C(!startResponse.HasError(), startResponse); + + log << TLOG_INFO << "Wait for write response"; + auto writeResponse = producer->Write(1, TString("blob1")).GetValueSync().Response; + UNIT_ASSERT(!writeResponse.HasError()); + + producerSettings.Server.UseLogbrokerCDS = cds ? EClusterDiscoveryUsageMode::Use : EClusterDiscoveryUsageMode::DontUse; + producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); + startResponse = producer->Start().GetValueSync().Response; + UNIT_ASSERT_C(!startResponse.HasError(), startResponse); + + log << TLOG_INFO << "Wait for write response"; + writeResponse = producer->Write(2, TString("blob2")).GetValueSync().Response; + UNIT_ASSERT(!writeResponse.HasError()); + } + + Y_UNIT_TEST(StartProducerWithCDS) { + StartProducerWithDiscovery(true); + } + + Y_UNIT_TEST(StartProducerWithoutCDS) { + StartProducerWithDiscovery(false); + } + + Y_UNIT_TEST(StartProducerWithCDSAndPreferAvailableCluster) { + StartProducerWithDiscovery(true, "dc1"); + } + + Y_UNIT_TEST(StartProducerWithCDSAndPreferUnavailableCluster) { + StartProducerWithDiscovery(true, "dc2", true); + } + + Y_UNIT_TEST(StartProducerWithCDSAndPreferUnknownCluster) { + SDKTestSetup setup{"PreferUnknownCluster"}; + auto producerSettings = setup.GetProducerSettings(); + producerSettings.Server.UseLogbrokerCDS = EClusterDiscoveryUsageMode::Use; + producerSettings.PreferredCluster = "blablabla"; + auto producer = setup.GetPQLib()->CreateProducer(producerSettings, nullptr, false); + auto startResponse = producer->Start().GetValueSync().Response; + UNIT_ASSERT_C(startResponse.HasError(), startResponse); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp new file mode 100644 index 0000000000..6e1c3f9b2e --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.cpp @@ -0,0 +1,31 @@ +#include "queue_pool.h" + +#include <util/digest/city.h> + +namespace NPersQueue { +void TQueuePool::Start(size_t queuesCount) { + Y_VERIFY(queuesCount > 0); + Queues.resize(queuesCount); + for (auto& queuePtr : Queues) { + queuePtr = std::make_shared<TThreadPool>(); + queuePtr->Start(1); // start one thread for each tag + } +} + +void TQueuePool::Stop() { + for (auto& queuePtr : Queues) { + queuePtr->Stop(); + } + Queues.clear(); +} + +const std::shared_ptr<IThreadPool>& TQueuePool::GetQueuePtr(const void* tag) { + Y_VERIFY(!Queues.empty()); + const size_t queue = static_cast<size_t>(CityHash64(reinterpret_cast<const char*>(&tag), sizeof(tag))) % Queues.size(); + return Queues[queue]; +} + +IThreadPool& TQueuePool::GetQueue(const void* tag) { + return *GetQueuePtr(tag); +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h new file mode 100644 index 0000000000..961b8ff446 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool.h @@ -0,0 +1,23 @@ +#pragma once +#include <util/generic/ptr.h> +#include <util/thread/pool.h> + +#include <functional> +#include <memory> +#include <vector> + +namespace NPersQueue { +class TQueuePool { +public: + void Start(size_t queuesCount); + void Stop(); + + // get one-thread-queue for processing specified tag (address) + IThreadPool& GetQueue(const void* tag); + + const std::shared_ptr<IThreadPool>& GetQueuePtr(const void* tag); + +private: + std::vector<std::shared_ptr<IThreadPool>> Queues; +}; +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp new file mode 100644 index 0000000000..9f8d792e52 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/queue_pool_ut.cpp @@ -0,0 +1,26 @@ +#include "queue_pool.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <unordered_map> + +namespace NPersQueue { +Y_UNIT_TEST_SUITE(TQueuePoolTest) { + Y_UNIT_TEST(QueuesDistribution) { + TQueuePool pool; + pool.Start(10); + size_t addresses[10000] = {}; + std::unordered_map<IThreadPool*, size_t> queueAddresses; + for (size_t& address : addresses) { + IThreadPool* q = &pool.GetQueue(&address); + ++queueAddresses[q]; + UNIT_ASSERT_EQUAL(q, &pool.GetQueue(&address)); // one address always leads to one queue + } + UNIT_ASSERT_VALUES_EQUAL(queueAddresses.size(), 10); + for (const auto& queueToCount : queueAddresses) { + UNIT_ASSERT_C(queueToCount.second >= 850, "Count: " << queueToCount.second); + UNIT_ASSERT_C(queueToCount.second <= 1150, "Count: " << queueToCount.second); + } + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp new file mode 100644 index 0000000000..02b3aafafc --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.cpp @@ -0,0 +1,526 @@ +#include "retrying_consumer.h" + +namespace { + bool IsRetryable(const NPersQueue::TError& err) { + switch (err.code()) { + case NPersQueue::NErrorCode::INITIALIZING: + case NPersQueue::NErrorCode::OVERLOAD: + case NPersQueue::NErrorCode::READ_TIMEOUT: + case NPersQueue::NErrorCode::TABLET_IS_DROPPED: + case NPersQueue::NErrorCode::CREATE_TIMEOUT: + case NPersQueue::NErrorCode::ERROR: + case NPersQueue::NErrorCode::CLUSTER_DISABLED: + return true; + default: + return false; + } + } +} + +namespace NPersQueue { + +TRetryingConsumer::TRetryingConsumer(const TConsumerSettings& settings, + std::shared_ptr<void> destroyEventRef, + TIntrusivePtr<TPQLibPrivate> pqLib, + TIntrusivePtr<ILogger> logger) + : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) + , Settings(settings) + , Logger(std::move(logger)) + , IsDeadPromise(NThreading::NewPromise<TError>()) + , ConsumerDestroyedPromise(NThreading::NewPromise<void>()) + , GenCounter(1) + , CookieCounter(1) + , ReconnectionAttemptsDone(0) + , Stopping(false) + , Reconnecting(false) +{ + Y_VERIFY(Settings.ReconnectOnFailure, "ReconnectOnFailure should be set."); + Y_ENSURE(Settings.MaxAttempts != 0, "MaxAttempts setting can't be zero."); + Y_ENSURE(Settings.ReconnectionDelay != TDuration::Zero(), "ReconnectionDelay setting can't be zero."); + Y_ENSURE(Settings.MaxReconnectionDelay != TDuration::Zero(), "MaxReconnectionDelay setting can't be zero."); + Y_ENSURE(Settings.StartSessionTimeout != TDuration::Zero(), "StartSessionTimeout setting can't be zero."); + + Settings.ReconnectOnFailure = false; // reset flag, to use the settings for creating sub consumer +} + +TRetryingConsumer::~TRetryingConsumer() noexcept { + Cancel(); +} + +NThreading::TFuture<TConsumerCreateResponse> TRetryingConsumer::Start(TInstant deadline) noexcept { + StartPromise = NThreading::NewPromise<TConsumerCreateResponse>(); + if (deadline != TInstant::Max()) { + std::weak_ptr<TRetryingConsumer> weakRef = shared_from_this(); + auto onDeadline = [weakRef] { + auto self = weakRef.lock(); + if (nullptr != self) { + self->OnStartDeadline(); + } + }; + PQLib->GetScheduler().Schedule(deadline, this, onDeadline); + } + DoReconnect(deadline); + return StartPromise.GetFuture(); +} + +NThreading::TFuture<TError> TRetryingConsumer::IsDead() noexcept { + return IsDeadPromise.GetFuture(); +} + +void TRetryingConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { + if (!StartPromise.Initialized() || Stopping) { + TReadResponse res; + res.MutableError()->SetDescription("consumer is not ready"); + res.MutableError()->SetCode(NErrorCode::ERROR); + promise.SetValue(TConsumerMessage{std::move(res)}); + return; + } + + if (!ReadyResponses.empty()) { + Y_ASSERT(PendingRequests.empty()); + promise.SetValue(std::move(ReadyResponses.front())); + ReadyResponses.pop_front(); + return; + } + + PendingRequests.push_back(promise); + + if (Consumer && StartFuture.HasValue()) { + DoRequest(); + } +} + +static void FormatCookies(TStringBuilder& ret, const TVector<ui64>& cookies) { + ret << "{"; + for (size_t i = 0; i < cookies.size(); ++i) { + if (i > 0) { + ret << ", "; + } + ret << cookies[i]; + } + ret << "}"; +} + +static TString FormatCommitForLog(const TStringBuf reason, const TVector<ui64>& cookies, const TVector<ui64>& originalCookies, const TVector<ui64>& committedCookies) { + TStringBuilder ret; + ret << "Commit cookies by retrying consumer" << reason << ". User cookies: "; + FormatCookies(ret, cookies); + ret << ". Subconsumer cookies to commit: "; + FormatCookies(ret, originalCookies); + ret << ". Skipped cookies: "; + FormatCookies(ret, committedCookies); + return std::move(ret); +} + +void TRetryingConsumer::Commit(const TVector<ui64>& cookies) noexcept { + if (!Consumer) { + if (!StartPromise.Initialized() || Stopping) { + Destroy("Not ready", NErrorCode::BAD_REQUEST); + } else { + // just response that cookies were commited + FastCommit(cookies); + DEBUG_LOG(FormatCommitForLog(". Consumer is not initialied", cookies, {}, cookies), "", SessionId); + } + return; + } + if (Settings.CommitsDisabled) { + Destroy("Commits are disabled", NErrorCode::BAD_REQUEST); + return; + } + + TVector<ui64> originalCookies(Reserve(cookies.size())); + // cookies which can be treated as committed + TVector<ui64> commitedCookies; + ui64 minCookie = CookieCounter - Cookies.size(); + for (auto cookie : cookies) { + if (cookie >= minCookie && cookie < CookieCounter) { + auto& cookieInfo = Cookies[cookie - minCookie]; + Y_VERIFY(cookieInfo.UserCookie == cookie); + if (0 != cookieInfo.OriginalCookie) { + CommittingCookies[cookieInfo.OriginalCookie] = cookie; + originalCookies.push_back(cookieInfo.OriginalCookie); + cookieInfo.OriginalCookie = 0; + for (auto* lock : cookieInfo.Locks) { + lock->Cookies.erase(cookie); + } + } else { + commitedCookies.push_back(cookie); + } + } else if (cookie >= CookieCounter) { + Destroy("Unknown cookie", NErrorCode::BAD_REQUEST); + break; + } else { + commitedCookies.push_back(cookie); + } + } + if (!originalCookies.empty()) { + Consumer->Commit(originalCookies); + } + if (!commitedCookies.empty()) { + FastCommit(commitedCookies); + } + DEBUG_LOG(FormatCommitForLog("", cookies, originalCookies, commitedCookies), "", SessionId); + // clean commited cookies + while (!Cookies.empty() && 0 == Cookies.front().OriginalCookie) { + Cookies.pop_front(); + } +} + +void TRetryingConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { + if (!Consumer) { + // client should receive message about release lock + return; + } + auto lockIt = Locks.find(std::make_pair(topic, partition)); + if (lockIt == Locks.end() || !lockIt->second.Locked) { + WARN_LOG("Requesting partition status on partition without lock. Topic: " << topic << ". Partition: " << partition, "", SessionId); + } else if (lockIt->second.Gen != generation) { + WARN_LOG("Requesting partition status on partition with wrong generation of lock. Topic: " << topic << ". Partition: " << partition << ". Generation: " << generation, "", SessionId); + } else { + Consumer->RequestPartitionStatus(topic, partition, lockIt->second.OriginalGen); + } +} + +void TRetryingConsumer::Cancel() { + Destroy(GetCancelReason()); +} + +NThreading::TFuture<void> TRetryingConsumer::Destroyed() noexcept { + return ConsumerDestroyedPromise.GetFuture(); +} + +void TRetryingConsumer::OnStartDeadline() { + if (!StartPromise.HasValue()) { + TError error; + error.SetDescription("Start timeout."); + error.SetCode(NErrorCode::CREATE_TIMEOUT); + Destroy(error); + } +} + +void TRetryingConsumer::OnConsumerDead(const TError& error) { + WARN_LOG("Subconsumer is dead: " << error, "", SessionId); + ScheduleReconnect(); +} + +void TRetryingConsumer::ScheduleReconnect() { + if (Stopping || Reconnecting) { + return; + } + Reconnecting = true; + if (!Locks.empty()) { + // need to notify client that all locks are expired + for (auto&& [key, lock] : Locks) { + if (lock.Locked) { + TReadResponse response; + auto* release = response.MutableRelease(); + release->set_generation(lock.Gen); + release->set_topic(key.first); + release->set_partition(key.second); + release->set_can_commit(false); + FastResponse(TConsumerMessage(std::move(response))); + } + } + } + if (!CommittingCookies.empty()) { + // need to notify client that all cookies are commited, because these cookies aren't valid anymore + TReadResponse response; + auto* cookies = response.MutableCommit()->MutableCookie(); + cookies->Reserve(CommittingCookies.size()); + for (const auto& cookiePair : CommittingCookies) { + cookies->Add(cookiePair.second); + } + FastResponse(TConsumerMessage(std::move(response))); + } + + Cookies.clear(); + CommittingCookies.clear(); + Locks.clear(); + Consumer = nullptr; + + if (ReconnectionAttemptsDone >= Settings.MaxAttempts) { + Destroy(TStringBuilder() << "Failed " << ReconnectionAttemptsDone << " reconnection attempts"); + return; + } + + ++ReconnectionAttemptsDone; + + TDuration delay = Min(Settings.MaxReconnectionDelay, ReconnectionAttemptsDone * Settings.ReconnectionDelay); + std::weak_ptr<TRetryingConsumer> weakRef = shared_from_this(); + PQLib->GetScheduler().Schedule(delay, this, [weakRef]() { + auto self = weakRef.lock(); + if (nullptr != self) { + self->DoReconnect(self->Settings.StartSessionTimeout.ToDeadLine()); + self->Reconnecting = false; + } + }); +} + +void TRetryingConsumer::DoReconnect(TInstant deadline) { + if (Stopping) { + return; + } + DEBUG_LOG("Create subconsumer", "", SessionId); + Consumer = PQLib->CreateRawConsumer(Settings, DestroyEventRef, Logger); + std::weak_ptr<TRetryingConsumer> weak(shared_from_this()); + StartFuture = Consumer->Start(deadline); + PQLib->Subscribe(StartFuture, this, [weak](const NThreading::TFuture<TConsumerCreateResponse>& f) { + auto self = weak.lock(); + if (nullptr != self) { + self->StartProcessing(f); + } + }); + + PQLib->Subscribe(Consumer->IsDead(), this, [weak](const NThreading::TFuture<TError>& error) { + auto self = weak.lock(); + if (nullptr != self) { + self->OnConsumerDead(error.GetValue()); + } + }); +} + +void TRetryingConsumer::StartProcessing(const NThreading::TFuture<TConsumerCreateResponse>& f) { + Y_VERIFY(f.HasValue()); + if (f.GetValue().Response.HasError()) { + WARN_LOG("Cannot create subconsumer: " << f.GetValue().Response.GetError(), "", SessionId); + if (IsRetryable(f.GetValue().Response.GetError())) { + ScheduleReconnect(); + } else { + Destroy(f.GetValue().Response.GetError()); + } + } else { + if (!SessionId) { + SessionId = f.GetValue().Response.GetInit().GetSessionId(); + } + ReconnectionAttemptsDone = 0; + // need to schedule again all pending requests which are running before connection lost + for (size_t cnt = PendingRequests.size(); 0 != cnt; --cnt) { + DoRequest(); + } + StartPromise.TrySetValue(f.GetValue()); + } +} + +void TRetryingConsumer::SubscribeDestroyed() { + NThreading::TPromise<void> promise = ConsumerDestroyedPromise; + auto handler = [promise](const auto&) mutable { + promise.SetValue(); + }; + if (Consumer) { + WaitExceptionOrAll(DestroyedPromise.GetFuture(), Consumer->Destroyed()).Subscribe(handler); + } else { + DestroyedPromise.GetFuture().Subscribe(handler); + } + + DestroyPQLibRef(); +} + +void TRetryingConsumer::Destroy(const TError& error) { + if (Stopping) { + return; + } + + Stopping = true; + + SubscribeDestroyed(); + + TConsumerMessage message = [&]() { + TReadResponse response; + *response.MutableError() = error; + return TConsumerMessage(std::move(response)); + }(); + + for (auto& r : PendingRequests) { + r.TrySetValue(message); + } + IsDeadPromise.TrySetValue(error); + if (StartPromise.Initialized()) { + StartPromise.TrySetValue(TConsumerCreateResponse(std::move(message.Response))); + } + PendingRequests.clear(); + ReadyResponses.clear(); + Cookies.clear(); + CommittingCookies.clear(); + Locks.clear(); + Consumer = nullptr; + Reconnecting = false; +} + +void TRetryingConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) { + TError error; + error.SetDescription(description); + error.SetCode(code); + Destroy(error); +} + +void TRetryingConsumer::DoRequest() { + Y_VERIFY(Consumer); + std::weak_ptr<TRetryingConsumer> weak(shared_from_this()); + PQLib->Subscribe(Consumer->GetNextMessage(), this, [weak](NThreading::TFuture<TConsumerMessage>& f) mutable { + auto self = weak.lock(); + if (nullptr != self) { + self->ProcessResponse(f.ExtractValueSync()); + } + }); +} + +void TRetryingConsumer::ProcessResponse(TConsumerMessage&& message) { + switch (message.Type) { + case NPersQueue::EMessageType::EMT_DATA: + { + auto* data = message.Response.mutable_data(); + TCookieInfo* cookie = nullptr; + if (!Settings.CommitsDisabled) { + Cookies.push_back(TCookieInfo{{}, data->cookie(), CookieCounter++}); + cookie = &Cookies.back(); + data->set_cookie(cookie->UserCookie); + DEBUG_LOG("Got data from subconsumer. Cookie: " << cookie->OriginalCookie << ". User cookie: " << cookie->UserCookie, "", SessionId); + } + + for (auto& b : data->message_batch()) { + auto lockIt = Locks.find(std::make_pair(b.topic(), b.partition())); + if (lockIt != Locks.end()) { + Y_VERIFY(lockIt->second.Locked); + if (!Settings.CommitsDisabled) { + Y_ASSERT(nullptr != cookie); + cookie->Locks.emplace(&lockIt->second); + lockIt->second.Cookies.emplace(cookie->UserCookie); + } + + // Validate that all offsets are >= LockInfo.ReadOffset as expected. + if (lockIt->second.ReadOffset != 0) { + for (const auto& message : b.message()) { + if (message.offset() < lockIt->second.ReadOffset) { + Destroy( + TStringBuilder() + << "Fatal error: expected offsets for topic " << b.topic() + << " partition " << b.partition() << " >= " << lockIt->second.ReadOffset + << ", but got offset " << message.offset() + ); + return; + } + } + } + } + } + // TODO (bulatman) what about batched_data? + } + break; + case NPersQueue::EMessageType::EMT_LOCK: + { + auto* lock = message.Response.mutable_lock(); + auto& lockInfo = Locks[std::make_pair(lock->topic(), lock->partition())]; + Y_VERIFY(!lockInfo.Locked); + lockInfo.Locked = true; + lockInfo.Gen = GenCounter++; + lockInfo.OriginalGen = lock->generation(); + + lock->set_generation(lockInfo.Gen); + + DEBUG_LOG("Got lock from subconsumer on (" << lock->topic() << ", " << lock->partition() << "). Generation: " << lockInfo.OriginalGen << ". User generation: " << lockInfo.Gen, "", SessionId); + + std::weak_ptr<TRetryingConsumer> weak = shared_from_this(); + PQLib->Subscribe(message.ReadyToRead.GetFuture(), this, + [weak, topic = lock->topic(), partition = lock->partition(), generation = lock->generation()](const NThreading::TFuture<NPersQueue::TLockInfo>& lockInfo) { + auto self = weak.lock(); + if (nullptr != self) { + self->UpdateReadyToRead(lockInfo.GetValue(), topic, partition, generation); + } + }); + } + break; + case NPersQueue::EMessageType::EMT_RELEASE: + { + auto* release = message.Response.mutable_release(); + const bool softRelease = release->can_commit(); + auto lockIt = Locks.find(std::make_pair(release->topic(), release->partition())); + const bool lockInfoFound = lockIt != Locks.end(); + DEBUG_LOG("Got release from subconsumer on (" << release->topic() << ", " << release->partition() << "). Can commit: " << softRelease << ". Has lock info: " << lockInfoFound << ". Generation: " << release->generation() << ". User generation: " << (lockInfoFound ? lockIt->second.Gen : 0), "", SessionId); + if (lockInfoFound) { // It is normal situation when client receives Release(canCommit=true) and then Release(canCommit=false). + auto& lockInfo = lockIt->second; + Y_VERIFY(lockInfo.OriginalGen == release->generation(), "lock generation mismatch"); + release->set_generation(lockInfo.Gen); + + if (softRelease) { + lockInfo.Locked = false; + } else { + for (auto cookie : lockInfo.Cookies) { + auto& cookieInfo = Cookies[cookie + Cookies.size() - CookieCounter]; + Y_VERIFY(cookieInfo.UserCookie == cookie); + // Lock is not valid anymore. + cookieInfo.Locks.erase(&lockInfo); + } + Locks.erase(lockIt); + } + } else { + return; + } + } + break; + case NPersQueue::EMessageType::EMT_STATUS: + { + auto* status = message.Response.mutable_partition_status(); + auto lockIt = Locks.find(std::make_pair(status->topic(), status->partition())); + Y_VERIFY(lockIt != Locks.end() && lockIt->second.Locked && lockIt->second.OriginalGen == status->generation()); + status->set_generation(lockIt->second.Gen); + } + break; + case NPersQueue::EMessageType::EMT_COMMIT: + { + auto* cookies = message.Response.mutable_commit()->mutable_cookie(); + // convert cookies + for (int i = 0; i < cookies->size(); ++i) { + auto it = CommittingCookies.find(cookies->Get(i)); + Y_VERIFY(it != CommittingCookies.end(), "unknown commited cookie!"); + cookies->Set(i, it->second); + CommittingCookies.erase(it); + } + } + break; + + case NPersQueue::EMessageType::EMT_ERROR: + // check error, if retryable, need to recreate consumer and read message again + const bool retryable = IsRetryable(message.Response.error()); + DEBUG_LOG("Got error from subconsumer: " << message.Response.error() << ". Retryable: " << retryable, "", SessionId); + if (retryable) { + ScheduleReconnect(); + return; + } + break; + }; + + if (!PendingRequests.empty()) { + PendingRequests.front().SetValue(std::move(message)); + PendingRequests.pop_front(); + } else { + ReadyResponses.push_back(std::move(message)); + } +} + +void TRetryingConsumer::FastResponse(TConsumerMessage&& message) { + if (!PendingRequests.empty()) { + PendingRequests.front().SetValue(std::move(message)); + PendingRequests.pop_front(); + } else { + ReadyResponses.push_back(std::move(message)); + } +} + +void TRetryingConsumer::FastCommit(const TVector<ui64>& cookies) { + TReadResponse response; + auto* cookiesMessage = response.MutableCommit()->MutableCookie(); + cookiesMessage->Reserve(cookies.size()); + for (auto cookie : cookies) { + cookiesMessage->Add(cookie); + } + FastResponse(TConsumerMessage(std::move(response))); +} + +void TRetryingConsumer::UpdateReadyToRead(const NPersQueue::TLockInfo& readyToRead, const TString& topic, ui32 partition, ui64 generation) { + const auto lockIt = Locks.find(std::make_pair(topic, partition)); + if (lockIt != Locks.end() && lockIt->second.Locked && lockIt->second.Gen == generation) { + lockIt->second.ReadOffset = readyToRead.ReadOffset; + } +} + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h new file mode 100644 index 0000000000..448d35e4c8 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer.h @@ -0,0 +1,107 @@ +#pragma once + +#include "consumer.h" +#include "scheduler.h" +#include "internals.h" +#include "persqueue_p.h" +#include "iconsumer_p.h" + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> + +#include <library/cpp/threading/future/future.h> + +#include <util/generic/hash.h> +#include <util/generic/hash_set.h> +#include <util/generic/vector.h> + +#include <deque> + +namespace NPersQueue { + +// @brief consumer implementation which transparently retries all connectivity errors +class TRetryingConsumer: public IConsumerImpl, public std::enable_shared_from_this<TRetryingConsumer> { + // @brief locked partitions info + struct TLockInfo { + THashSet<ui64> Cookies; // related cookies + ui64 Gen = 0; + ui64 OriginalGen = 0; // original generation + ui64 ReadOffset = 0; // read offset specified by client + bool Locked = false; + }; + + struct TCookieInfo { + THashSet<TLockInfo*> Locks; // related locks + ui64 OriginalCookie = 0; // zero means invalid cookie + ui64 UserCookie = 0; + }; + +public: + TRetryingConsumer(const TConsumerSettings& settings, std::shared_ptr<void> destroyEventRef, + TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger); + + ~TRetryingConsumer() noexcept override; + + NThreading::TFuture<TConsumerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept override; + + using IConsumerImpl::GetNextMessage; + void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; + + void Commit(const TVector<ui64>& cookies) noexcept override; + + void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; + + NThreading::TFuture<TError> IsDead() noexcept override; + + void Cancel() override; + + NThreading::TFuture<void> Destroyed() noexcept override; + +private: + void OnStartDeadline(); + void OnConsumerDead(const TError& error); + void ScheduleReconnect(); + void DoReconnect(TInstant deadline); + void StartProcessing(const NThreading::TFuture<TConsumerCreateResponse>& f); + void SubscribeDestroyed(); + void Destroy(const TError& error); + void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); + void DoRequest(); + void ProcessResponse(TConsumerMessage&& message); + void FastResponse(TConsumerMessage&& message); + void FastCommit(const TVector<ui64>& cookies); + void UpdateReadyToRead(const NPersQueue::TLockInfo& readyToRead, const TString& topic, ui32 partition, ui64 generation); + +private: + TConsumerSettings Settings; + TIntrusivePtr<ILogger> Logger; + std::shared_ptr<IConsumerImpl> Consumer; + TString SessionId; + + NThreading::TFuture<TConsumerCreateResponse> StartFuture; + NThreading::TPromise<TConsumerCreateResponse> StartPromise; + NThreading::TPromise<TError> IsDeadPromise; + NThreading::TPromise<void> ConsumerDestroyedPromise; + + // requests which are waiting for response + std::deque<NThreading::TPromise<TConsumerMessage>> PendingRequests; + // ready messages for returning to clients immediately + std::deque<TConsumerMessage> ReadyResponses; + // active cookies + std::deque<TCookieInfo> Cookies; + THashMap<ui64, ui64> CommittingCookies; + // active data per topic, partition + THashMap<std::pair<TString, ui64>, TLockInfo> Locks; // topic, partition -> LockInfo + + // number of unsuccessful retries after last error + ui64 GenCounter; + ui64 CookieCounter; + unsigned ReconnectionAttemptsDone; + // destroying process started + bool Stopping; + // reconnecting in process + bool Reconnecting; +}; +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp new file mode 100644 index 0000000000..8065ed717a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_consumer_ut.cpp @@ -0,0 +1,604 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/guid.h> + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + +namespace NPersQueue { + Y_UNIT_TEST_SUITE(TRetryingConsumerTest) { + static TConsumerSettings FakeSettings() { + TConsumerSettings settings; + settings.ReconnectOnFailure = true; + settings.Server = TServerSetting{"localhost"}; + settings.Topics.push_back("topic"); + settings.ClientId = "client"; + return settings; + } + + Y_UNIT_TEST(NotStartedConsumerCanBeDestructed) { + // Test that consumer doesn't hang on till shutdown + TPQLib lib; + auto settings = FakeSettings(); + lib.CreateConsumer(settings); + } + + Y_UNIT_TEST(StartDeadlineExpires) { + TPQLibSettings libSettings; + libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); + libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG); + TPQLib lib(libSettings); + THolder<IConsumer> consumer; + { + auto settings = FakeSettings(); + settings.ReconnectionDelay = TDuration::MilliSeconds(1); + settings.StartSessionTimeout = TDuration::MilliSeconds(10); + if (GrpcV1EnabledByDefault()) { + settings.MaxAttempts = 3; + } + consumer = lib.CreateConsumer(settings); + } + const TInstant beforeStart = TInstant::Now(); + auto future = consumer->Start(TDuration::MilliSeconds(100)); + if (!GrpcV1EnabledByDefault()) { + UNIT_ASSERT(future.GetValueSync().Response.HasError()); + UNIT_ASSERT_EQUAL_C(future.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, + "Error: " << future.GetValueSync().Response.GetError()); + const TInstant now = TInstant::Now(); + UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(100), now); + } + auto isDead = consumer->IsDead(); + isDead.Wait(); + + DestroyAndWait(consumer); + } + + Y_UNIT_TEST(StartMaxAttemptsExpire) { + TPQLibSettings libSettings; + libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); + libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG); + TPQLib lib(libSettings); + THolder<IConsumer> consumer; + { + auto settings = FakeSettings(); + settings.ReconnectionDelay = TDuration::MilliSeconds(10); + settings.StartSessionTimeout = TDuration::MilliSeconds(10); + settings.MaxAttempts = 3; + consumer = lib.CreateConsumer(settings); + } + const TInstant beforeStart = TInstant::Now(); + auto future = consumer->Start(); + if (!GrpcV1EnabledByDefault()) { + UNIT_ASSERT(future.GetValueSync().Response.HasError()); + } + auto deadFuture = consumer->IsDead(); + deadFuture.Wait(); + const TInstant now = TInstant::Now(); + UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(30), now); + + DestroyAndWait(consumer); + } + + static void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { + const TProducerCommitResponse& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); + } + + static void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); + } + + static void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response); + } + + static void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response); + } + + static TProducerSettings MakeProducerSettings(const TTestServer& testServer) { + TProducerSettings producerSettings; + producerSettings.ReconnectOnFailure = false; + producerSettings.Topic = "topic1"; + producerSettings.SourceId = "123"; + producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + producerSettings.Codec = ECodec::LZOP; + return producerSettings; + } + + static TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { + TConsumerSettings settings; + settings.ReconnectOnFailure = true; + settings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + settings.Topics.emplace_back("topic1"); + settings.ClientId = "user"; + return settings; + } + + Y_UNIT_TEST(ReconnectsToServer) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + const size_t partitions = 10; + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + + testServer.WaitInit("topic1"); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLib PQLib; + + auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); + UNIT_ASSERT(!producer->IsDead().HasValue()); + producer->Start().Wait(); + + auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); + TFuture<TError> isDead = consumer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + consumer->Start().Wait(); + + auto read1 = consumer->GetNextMessage(); + + // write first value + auto write1 = producer->Write(1, TString("blob1")); + AssertWriteValid(write1); + AssertReadValid(read1); + + // commit + consumer->Commit({1}); + auto commitAck = consumer->GetNextMessage(); + Cerr << "wait for commit1\n"; + + AssertCommited(commitAck); + + Cerr << "After shutdown\n"; + testServer.ShutdownGRpc(); + + auto read2 = consumer->GetNextMessage(); + UNIT_ASSERT(!isDead.HasValue()); + testServer.EnableGRpc(); + + testServer.WaitInit("topic1"); + + Cerr << "Wait producer1 death\n"; + + producer->IsDead().Wait(); + + producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); + Cerr << "Wait producer2 start\n"; + producer->Start().Wait(); + UNIT_ASSERT(!producer->IsDead().HasValue()); + auto write2 = producer->Write(2, TString("blob2")); + Cerr << "Wait for second write\n"; + AssertWriteValid(write2); + Cerr << "wait for read second blob\n"; + AssertReadValid(read2); + UNIT_ASSERT_VALUES_EQUAL(2u, read2.GetValueSync().Response.GetData().GetCookie()); + UNIT_ASSERT(!isDead.HasValue()); + consumer->Commit({2}); + Cerr << "wait for commit2\n"; + auto commitAck2 = consumer->GetNextMessage(); + AssertCommited(commitAck2); + + DestroyAndWait(producer); + DestroyAndWait(consumer); + } + + static void DiesOnTooManyReconnectionAttempts(bool callRead) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2); + + testServer.WaitInit("topic1"); + + TConsumerSettings settings = MakeConsumerSettings(testServer); + settings.MaxAttempts = 3; + settings.ReconnectionDelay = TDuration::MilliSeconds(100); + auto consumer = testServer.PQLib->CreateConsumer(settings, testServer.PQLibSettings.DefaultLogger, false); + consumer->Start().Wait(); + TFuture<TError> isDead = consumer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + // shutdown server + const TInstant beforeShutdown = TInstant::Now(); + testServer.ShutdownServer(); + + NThreading::TFuture<TConsumerMessage> read; + if (callRead) { + read = consumer->GetNextMessage(); + } + + isDead.Wait(); + const TInstant afterDead = TInstant::Now(); + // 3 attempts: 100ms, 200ms and 300ms + UNIT_ASSERT_C(afterDead - beforeShutdown >= TDuration::MilliSeconds(GrpcV1EnabledByDefault() ? 300 : 600), "real difference: " << (afterDead - beforeShutdown)); + + if (callRead) { + AssertReadFailed(read); + } + } + + Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithoutRead) { + // Check that we reconnect even without explicit write errors + DiesOnTooManyReconnectionAttempts(false); + } + + Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithRead) { + DiesOnTooManyReconnectionAttempts(true); + } + + Y_UNIT_TEST(ReadBeforeConnectToServer) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + const size_t partitions = 10; + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + + testServer.WaitInit("topic1"); + + auto consumer = testServer.PQLib->CreateConsumer( + MakeConsumerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false + ); + auto consumerStarted = consumer->Start(); + auto read1 = consumer->GetNextMessage(); + consumerStarted.Wait(); + TVector<TFuture<TConsumerMessage>> messages; + for (int i = 2; i <= 5; ++i) { + messages.push_back(consumer->GetNextMessage()); + } + + auto producer = testServer.PQLib->CreateProducer( + MakeProducerSettings(testServer), testServer.PQLibSettings.DefaultLogger, false + ); + producer->Start().Wait(); + + while (!messages.back().HasValue()) { + auto write = producer->Write(TString("data")); + AssertWriteValid(write); + } + + AssertReadValid(read1); + for (auto& msg: messages) { + AssertReadValid(msg); + } + + DestroyAndWait(producer); + DestroyAndWait(consumer); + } + + Y_UNIT_TEST(CancelsStartAfterPQLibDeath) { + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLibSettings pqLibSettings; + pqLibSettings.DefaultLogger = logger; + THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); + + auto consumer = PQLib->CreateConsumer(FakeSettings(), logger, false); + auto start = consumer->Start(); + if (!GrpcV1EnabledByDefault()) { + UNIT_ASSERT(!start.HasValue()); + } + + PQLib = nullptr; + + if (!GrpcV1EnabledByDefault()) { + UNIT_ASSERT(start.HasValue()); + UNIT_ASSERT(start.GetValue().Response.HasError()); + } + + auto dead = consumer->IsDead(); + if (GrpcV1EnabledByDefault()) { + dead.Wait(); + } else { + UNIT_ASSERT(dead.HasValue()); + } + + auto read = consumer->GetNextMessage(); + UNIT_ASSERT(read.HasValue()); + UNIT_ASSERT(read.GetValueSync().Response.HasError()); + } + + Y_UNIT_TEST(LockSession) { + SDKTestSetup setup("LockSession", false); + setup.GetGrpcServerOptions().SetGRpcShutdownDeadline(TDuration::MilliSeconds(100)); + setup.Start(); + auto consumerSettings = setup.GetConsumerSettings(); + consumerSettings.UseLockSession = true; + consumerSettings.ReconnectOnFailure = true; + auto consumer = setup.StartConsumer(consumerSettings); + + setup.WriteToTopic({"msg1", "msg2"}); + + size_t releases = 0; + for (size_t times = 0; times < 2; ++times) { + while (true) { + auto msgFuture = consumer->GetNextMessage(); + auto& msg = msgFuture.GetValueSync(); + setup.GetLog() << TLOG_INFO << "Got response from consumer: " << msg.Response; + if (times) { + UNIT_ASSERT_C(msg.Type == EMT_LOCK || msg.Type == EMT_DATA || msg.Type == EMT_RELEASE, msg.Response); + } else { + UNIT_ASSERT_C(msg.Type == EMT_LOCK || msg.Type == EMT_DATA, msg.Response); + } + if (msg.Type == EMT_LOCK) { + TLockInfo lockInfo; + lockInfo.ReadOffset = 1; + msg.ReadyToRead.SetValue(lockInfo); + } else if (msg.Type == EMT_DATA) { + UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().MessageBatchSize(), 1, msg.Response); + UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().GetMessageBatch(0).MessageSize(), 1, msg.Response); + UNIT_ASSERT_VALUES_EQUAL_C(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg2", msg.Response); + break; + } else if (msg.Type == EMT_RELEASE) { + ++releases; + } + } + if (!times) { // force reconnect + setup.ShutdownGRpc(); + setup.EnableGRpc(); + } + } + UNIT_ASSERT_VALUES_EQUAL(releases, 1); + } + + Y_UNIT_TEST(ReadAhead) { + SDKTestSetup setup("ReadAhead"); + + auto consumerSettings = setup.GetConsumerSettings(); + consumerSettings.ReconnectOnFailure = true; + auto consumer = setup.StartConsumer(consumerSettings); + + setup.WriteToTopic({"msg"}); + + // Wait until read ahead occurs. + Sleep(TDuration::MilliSeconds(100)); + + setup.ReadFromTopic({{"msg"}}, true, consumer.Get()); + } + + Y_UNIT_TEST(ReadBeforeWrite) { + SDKTestSetup setup("ReadBeforeWrite"); + + auto consumerSettings = setup.GetConsumerSettings(); + consumerSettings.ReconnectOnFailure = true; + auto consumer = setup.StartConsumer(consumerSettings); + + auto msgFuture = consumer->GetNextMessage(); + + Sleep(TDuration::MilliSeconds(100)); + + setup.WriteToTopic({"msg"}); + + const auto& msg = msgFuture.GetValueSync(); + UNIT_ASSERT_EQUAL(msg.Type, EMT_DATA); + UNIT_ASSERT_STRINGS_EQUAL(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg"); + } + + Y_UNIT_TEST(CommitDisabled) { + SDKTestSetup setup("CommitDisabled"); + + auto consumerSettings = setup.GetConsumerSettings(); + consumerSettings.ReconnectOnFailure = true; + consumerSettings.CommitsDisabled = true; + auto consumer = setup.StartConsumer(consumerSettings); + setup.WriteToTopic({"msg", "msg2"}); + setup.ReadFromTopic({{"msg", "msg2"}}, false, consumer.Get()); + // check that commit fails with error when commits disabled + setup.WriteToTopic({"msg3"}); + auto msgFuture = consumer->GetNextMessage(); + const auto& msg = msgFuture.GetValueSync(); + UNIT_ASSERT_C(!msg.Response.HasError(), msg.Response); + UNIT_ASSERT_EQUAL(msg.Type, EMT_DATA); + UNIT_ASSERT_STRINGS_EQUAL(msg.Response.GetData().GetMessageBatch(0).GetMessage(0).GetData(), "msg3"); + consumer->Commit({msg.Response.GetData().GetCookie()}); + auto isDead = consumer->IsDead(); + auto& err = isDead.GetValueSync(); + UNIT_ASSERT_STRINGS_EQUAL("Commits are disabled", err.GetDescription()); + } + + void LockReleaseTest(bool multiclusterConsumer, bool killPqrb) { + SDKTestSetup setup("LockRelease"); + const TString topicName = "lock_release"; + const size_t partitionsCount = 10; + setup.CreateTopic(topicName, setup.GetLocalCluster(), partitionsCount); + + auto consumerSettings = setup.GetConsumerSettings(); + consumerSettings.Topics[0] = topicName; + consumerSettings.ReconnectOnFailure = true; + consumerSettings.UseLockSession = true; + if (multiclusterConsumer) { // The same test for multicluster wrapper. + consumerSettings.ReadFromAllClusterSources = true; + consumerSettings.ReadMirroredPartitions = false; + } + auto consumer = setup.StartConsumer(consumerSettings); + + std::vector<int> partitionsLocked(partitionsCount * 2); // For both consumers. + std::vector<ui64> partitionsGens(partitionsCount * 2); + for (size_t i = 0; i < partitionsCount; ++i) { + auto lock = consumer->GetNextMessage().GetValueSync(); + UNIT_ASSERT_EQUAL_C(lock.Type, EMT_LOCK, lock.Response); + UNIT_ASSERT_C(!partitionsLocked[lock.Response.GetLock().GetPartition()], lock.Response); + partitionsLocked[lock.Response.GetLock().GetPartition()] = 1; + partitionsGens[lock.Response.GetLock().GetPartition()] = lock.Response.GetLock().GetGeneration(); + lock.ReadyToRead.SetValue({}); + } + + const size_t messagesCount = 100; + auto generateMessages = [&]() { + for (size_t i = 0; i < messagesCount; ++i) { // Write to several random partitions. + const TString uniqStr = CreateGuidAsString(); + auto settings = setup.GetProducerSettings(); + settings.Topic = topicName; + settings.SourceId = uniqStr; + auto producer = setup.StartProducer(settings); + setup.WriteToTopic({uniqStr}, producer.Get()); + } + }; + + generateMessages(); + + THolder<IConsumer> otherConsumer = nullptr; + bool rebalanced = false; + + NThreading::TFuture<TConsumerMessage> msg1 = consumer->GetNextMessage(); + NThreading::TFuture<TConsumerMessage> msg2; + size_t hardReleasesCount = 0; + + THashSet<TString> msgsReceived; + THashSet<ui64> cookiesToCommit; + auto locksCount = [&]() -> int { + return std::accumulate(partitionsLocked.begin(), partitionsLocked.end(), 0); + }; + auto allEventsReceived = [&]() -> bool { + return msgsReceived.size() >= messagesCount * 2 + && locksCount() == partitionsCount + && (!killPqrb || hardReleasesCount > 0) + && cookiesToCommit.empty(); + }; + + int iterationNum = 0; + while (!allEventsReceived()) { + ++iterationNum; + Cerr << "\nIterationNum: " << iterationNum << Endl; + Cerr << "msgsReceived.size(): " << msgsReceived.size() << Endl; + Cerr << "partitionsCount: " << partitionsCount << Endl; + Cerr << "locksGot: " << locksCount() << Endl; + Cerr << "hardReleasesCount: " << hardReleasesCount << Endl; + Cerr << "cookiesToCommit.size(): " << cookiesToCommit.size() << Endl; + auto waiter = msg2.Initialized() ? NThreading::WaitAny(msg1.IgnoreResult(), msg2.IgnoreResult()) : msg1.IgnoreResult(); + waiter.Wait(); + auto processValue = [&](auto& msg, auto& consumer) { + const bool isOtherConsumer = consumer == otherConsumer; + const ui64 consumerMix = isOtherConsumer ? 0x8000000000000000 : 0; + + switch (msg.GetValue().Type) { + case EMT_LOCK: + { + UNIT_ASSERT_LT(msg.GetValue().Response.GetLock().GetPartition(), partitionsCount); + const size_t idx = msg.GetValue().Response.GetLock().GetPartition() + (isOtherConsumer ? partitionsCount : 0); + if (msg.GetValue().Response.GetLock().GetGeneration() >= partitionsGens[idx]) { + UNIT_ASSERT_C(!partitionsLocked[idx], msg.GetValue().Response << ". Other: " << isOtherConsumer); + partitionsLocked[idx] = 1; + msg.GetValue().ReadyToRead.SetValue({}); + partitionsGens[idx] = msg.GetValue().Response.GetLock().GetGeneration(); + } + break; + } + case EMT_RELEASE: + { + UNIT_ASSERT_LT(msg.GetValue().Response.GetRelease().GetPartition(), partitionsCount); + const size_t idx = msg.GetValue().Response.GetRelease().GetPartition() + (isOtherConsumer ? partitionsCount : 0); + partitionsLocked[idx] = 0; + if (!msg.GetValue().Response.GetRelease().GetCanCommit()) { + ++hardReleasesCount; + } + if (!killPqrb) { + UNIT_ASSERT(msg.GetValue().Response.GetRelease().GetCanCommit()); // No restarts => soft release. + } + break; + } + case EMT_DATA: + { + const auto& resp = msg.GetValue().Response.GetData(); + UNIT_ASSERT(cookiesToCommit.insert(resp.GetCookie() | consumerMix).second); + for (const auto& batch : resp.GetMessageBatch()) { + for (const auto& message : batch.GetMessage()) { + msgsReceived.insert(message.GetData()); + } + } + consumer->Commit({resp.GetCookie()}); + break; + } + case EMT_ERROR: + case EMT_STATUS: + case EMT_COMMIT: + for (ui64 cookie : msg.GetValue().Response.GetCommit().GetCookie()) { + UNIT_ASSERT(cookiesToCommit.find(cookie | consumerMix) != cookiesToCommit.end()); + cookiesToCommit.erase(cookie | consumerMix); + } + break; + } + msg = consumer->GetNextMessage(); + }; + + if (msg1.HasValue()) { + processValue(msg1, consumer); + } + if (msg2.Initialized() && msg2.HasValue()) { + processValue(msg2, otherConsumer); + } + + if (!otherConsumer && msgsReceived.size() >= messagesCount / 4) { + otherConsumer = setup.StartConsumer(consumerSettings); + msg2 = otherConsumer->GetNextMessage(); + } + + if (!rebalanced && msgsReceived.size() >= messagesCount / 2) { + rebalanced = true; + + if (killPqrb) { + Sleep(TDuration::MilliSeconds(100)); + setup.KillPqrb(topicName, setup.GetLocalCluster()); + } + + generateMessages(); + } + } + + UNIT_ASSERT_VALUES_EQUAL(locksCount(), partitionsCount); + if (killPqrb) { + UNIT_ASSERT(hardReleasesCount > 0); + } else { + UNIT_ASSERT_VALUES_EQUAL(hardReleasesCount, 0); + } + } + + Y_UNIT_TEST(LockRelease) { + LockReleaseTest(false, false); + } + + Y_UNIT_TEST(LockReleaseHardRelease) { + LockReleaseTest(false, true); + } + + Y_UNIT_TEST(LockReleaseMulticluster) { + LockReleaseTest(true, false); + } + + Y_UNIT_TEST(LockReleaseMulticlusterHardRelease) { + LockReleaseTest(true, true); + } + + Y_UNIT_TEST(NonRetryableErrorOnStart) { + SDKTestSetup setup("NonRetryableErrorOnStart"); + + auto consumerSettings = setup.GetConsumerSettings(); + //consumerSettings. + consumerSettings.ReconnectOnFailure = true; + consumerSettings.Topics = { "unknown/topic" }; + THolder<IConsumer> consumer = setup.GetPQLib()->CreateConsumer(consumerSettings); + auto startFuture = consumer->Start(); + if (!GrpcV1EnabledByDefault()) { + Cerr << "===Got response: " << startFuture.GetValueSync().Response << Endl; + UNIT_ASSERT_C(startFuture.GetValueSync().Response.HasError(), startFuture.GetValueSync().Response); + UNIT_ASSERT_EQUAL(startFuture.GetValueSync().Response.GetError().GetCode(), NPersQueue::NErrorCode::UNKNOWN_TOPIC); + } + auto deadFuture = consumer->IsDead(); + UNIT_ASSERT_EQUAL_C(deadFuture.GetValueSync().GetCode(), NPersQueue::NErrorCode::UNKNOWN_TOPIC, deadFuture.GetValueSync()); + } + } +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp new file mode 100644 index 0000000000..bb562ea0ae --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.cpp @@ -0,0 +1,370 @@ +#include "retrying_producer.h" +#include "persqueue_p.h" + +#include <util/generic/strbuf.h> +#include <util/stream/zlib.h> +#include <util/string/cast.h> +#include <util/string/printf.h> +#include <util/string/vector.h> +#include <util/string/builder.h> + +namespace NPersQueue { + +TRetryingProducer::TRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> destroyEventRef, + TIntrusivePtr<TPQLibPrivate> pqLib, TIntrusivePtr<ILogger> logger) + : IProducerImpl(std::move(destroyEventRef), std::move(pqLib)) + , Settings(settings) + , Logger(std::move(logger)) + , IsDeadPromise(NThreading::NewPromise<TError>()) + , NeedRecreation(true) + , Stopping(false) + , ToProcess(0) + , LastReconnectionDelay(TDuration::Zero()) + , ReconnectionAttemptsDone(0) +{ + if (Settings.MaxAttempts == 0) { + ythrow yexception() << "MaxAttempts setting can't be zero."; + } + + if (Settings.ReconnectionDelay == TDuration::Zero()) { + ythrow yexception() << "ReconnectionDelay setting can't be zero."; + } + + if (Settings.StartSessionTimeout < PQLib->GetSettings().ChannelCreationTimeout) { + ythrow yexception() << "StartSessionTimeout can't be less than ChannelCreationTimeout."; + } +} + +TRetryingProducer::~TRetryingProducer() noexcept { + Destroy("Destructor called"); +} + +NThreading::TFuture<TProducerCreateResponse> TRetryingProducer::Start(TInstant deadline) noexcept { + StartPromise = NThreading::NewPromise<TProducerCreateResponse>(); + if (deadline != TInstant::Max()) { + std::weak_ptr<TRetryingProducer> self = shared_from_this(); + auto onDeadline = [self] { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnStartDeadline(); + } + }; + PQLib->GetScheduler().Schedule(deadline, this, onDeadline); + } + RecreateProducer(deadline); + return StartPromise.GetFuture(); +} + +void TRetryingProducer::RecreateProducer(TInstant deadline) noexcept { + Producer = nullptr; + if (Stopping) + return; + + Y_VERIFY(InFlightRequests.size() == Futures.size()); + DEBUG_LOG("Recreating subproducer. Futures size: " << Futures.size(), Settings.SourceId, ""); + if (Futures.empty()) { + DoRecreate(deadline); + } // otherwise it will be recreated in ProcessFutures() +} + +void TRetryingProducer::DoRecreate(TInstant deadline) noexcept { + Y_VERIFY(InFlightRequests.size() == Futures.size()); + Y_VERIFY(Futures.empty()); + if (Stopping) + return; + Y_VERIFY(NeedRecreation); + NeedRecreation = false; + if (ReconnectionAttemptsDone >= Settings.MaxAttempts) { + Destroy(TStringBuilder() << "Failed " << ReconnectionAttemptsDone << " reconnection attempts"); + return; + } + ++ReconnectionAttemptsDone; + DEBUG_LOG("Creating subproducer. Attempt: " << ReconnectionAttemptsDone, Settings.SourceId, ""); + Producer = PQLib->CreateRawProducer(Settings, DestroyEventRef, Logger); + StartFuture = Producer->Start(deadline); + + std::weak_ptr<TRetryingProducer> self(shared_from_this()); + PQLib->Subscribe(StartFuture, + this, + [self](const NThreading::TFuture<TProducerCreateResponse>& f) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->ProcessStart(f); + } + }); +} + +TDuration TRetryingProducer::UpdateReconnectionDelay() { + if (LastReconnectionDelay == TDuration::Zero()) { + LastReconnectionDelay = Settings.ReconnectionDelay; + } else { + LastReconnectionDelay *= 2; + } + LastReconnectionDelay = Min(LastReconnectionDelay, Settings.MaxReconnectionDelay); + return LastReconnectionDelay; +} + +void TRetryingProducer::ScheduleRecreation() { + if (Stopping) { + return; + } + NeedRecreation = true; + if (!ReconnectionCallback) { + const TDuration delay = UpdateReconnectionDelay(); + DEBUG_LOG("Schedule subproducer recreation through " << delay, Settings.SourceId, ""); + std::weak_ptr<TRetryingProducer> self(shared_from_this()); + ReconnectionCallback = + PQLib->GetScheduler().Schedule(delay, + this, + [self, sourceId = Settings.SourceId, logger = Logger] { + auto selfShared = self.lock(); + WRITE_LOG("Subproducer recreation callback. self is " << (selfShared ? "OK" : "nullptr") << ", stopping: " << (selfShared ? selfShared->Stopping : false), sourceId, "", TLOG_DEBUG, logger); + if (selfShared) { + selfShared->ReconnectionCallback = nullptr; + if (!selfShared->Stopping) { + selfShared->RecreateProducer(TInstant::Now() + selfShared->Settings.StartSessionTimeout); + } + } + }); + } +} + +void TRetryingProducer::ProcessStart(const NThreading::TFuture<TProducerCreateResponse>& f) noexcept { + INFO_LOG("Subproducer start response: " << f.GetValue().Response, Settings.SourceId, ""); + if (Stopping) + return; + if (NeedRecreation) + return; + if (!StartFuture.HasValue()) + return; + if (StartFuture.GetValue().Response.HasError()) { + WARN_LOG("Subproducer start error: " << f.GetValue().Response, Settings.SourceId, ""); + ScheduleRecreation(); + } else { + LastReconnectionDelay = TDuration::Zero(); + ReconnectionAttemptsDone = 0; + + // recreate on dead + DEBUG_LOG("Subscribe on subproducer death", Settings.SourceId, ""); + std::weak_ptr<TRetryingProducer> self(shared_from_this()); + PQLib->Subscribe(Producer->IsDead(), + this, + [self](const NThreading::TFuture<TError>& error) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->OnProducerDead(error.GetValue()); + } + }); + + if (!StartPromise.HasValue()) { + StartPromise.SetValue(StartFuture.GetValue()); + } + + SendData(); + } +} + +void TRetryingProducer::OnProducerDead(const TError& error) { + WARN_LOG("Subproducer is dead: " << error, Settings.SourceId, ""); + ScheduleRecreation(); +} + +void TRetryingProducer::SendData() noexcept { + if (Stopping) + return; + ui64 maxSeqNo = StartFuture.GetValue().Response.GetInit().GetMaxSeqNo(); + std::deque<NThreading::TPromise<TProducerCommitResponse>> promises; + std::deque<TWriteData> values; + ui64 prevSeqNo = 0; + for (auto& d : ResendRequests) { + Y_VERIFY(d.SeqNo == 0 || d.SeqNo > prevSeqNo); + if (d.SeqNo != 0) { + prevSeqNo = d.SeqNo; + } + if (d.SeqNo != 0 && d.SeqNo <= maxSeqNo) { + promises.push_back(Promises.front()); + Promises.pop_front(); + values.push_back(d); + } else { + DelegateWriteAndSubscribe(d.SeqNo, std::move(d.Data)); + } + } + //Requests can be not checked - it is up to client + for (auto& d : Requests) { + DelegateWriteAndSubscribe(d.SeqNo, std::move(d.Data)); + } + ResendRequests.clear(); + Requests.clear(); + for (ui32 i = 0; i < promises.size(); ++i) { + TWriteResponse res; + res.MutableAck()->SetAlreadyWritten(true); + res.MutableAck()->SetSeqNo(values[i].SeqNo); + promises[i].SetValue(TProducerCommitResponse(values[i].SeqNo, std::move(values[i].Data), std::move(res))); + } +} + +void TRetryingProducer::Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept { + Y_VERIFY(data.IsEncoded()); + if (!StartFuture.Initialized()) { + TWriteResponse res; + res.MutableError()->SetDescription("producer is not ready"); + res.MutableError()->SetCode(NErrorCode::ERROR); + promise.SetValue(TProducerCommitResponse{seqNo, std::move(data), std::move(res)}); + return; + } + + Promises.push_back(promise); + + if (!StartFuture.HasValue() || NeedRecreation || Stopping || !Requests.empty() || !ResendRequests.empty()) { + Requests.emplace_back(seqNo, std::move(data)); + } else { + DelegateWriteAndSubscribe(seqNo, std::move(data)); + } +} + +void TRetryingProducer::DelegateWriteAndSubscribe(TProducerSeqNo seqNo, TData&& data) noexcept { + Y_VERIFY(InFlightRequests.size() == Futures.size()); + if (seqNo == 0) { + Futures.push_back(Producer->Write(data)); + } else { + Futures.push_back(Producer->Write(seqNo, data)); + } + InFlightRequests.emplace_back(seqNo, std::move(data)); + + std::weak_ptr<TRetryingProducer> self(shared_from_this()); + PQLib->Subscribe(Futures.back(), + this, + [self](const NThreading::TFuture<TProducerCommitResponse>&) { + auto selfShared = self.lock(); + if (selfShared) { + selfShared->ProcessFutures(); + } + }); +} + +NThreading::TFuture<TError> TRetryingProducer::IsDead() noexcept { + return IsDeadPromise.GetFuture(); +} + +void TRetryingProducer::ProcessFutures() noexcept { + NThreading::TPromise<TProducerCommitResponse> promise; + NThreading::TFuture<TProducerCommitResponse> future; + { + if (Stopping) { + return; + } + ++ToProcess; + while (ToProcess) { + if (Futures.empty() || !Futures.front().HasValue()) { + break; + } + + Y_VERIFY(InFlightRequests.size() == Futures.size()); + --ToProcess; + + const TProducerCommitResponse& response = Futures.front().GetValue(); + const TWriteData& writeData = InFlightRequests.front(); + Y_VERIFY(Promises.size() == Futures.size() + Requests.size() + ResendRequests.size()); + + if (NeedRecreation) { + ResendRequests.emplace_back(response.SeqNo, TData(writeData.Data)); + InFlightRequests.pop_front(); + Futures.pop_front(); + continue; + } + + if (Futures.front().GetValue().Response.HasError()) { + Y_VERIFY(!NeedRecreation); + ScheduleRecreation(); + WARN_LOG("Future response with error: " << response.Response, Settings.SourceId, ""); + ResendRequests.emplace_back(response.SeqNo, TData(writeData.Data)); + InFlightRequests.pop_front(); + Futures.pop_front(); + Producer = nullptr; + continue; + } + + promise = Promises.front(); + Promises.pop_front(); + future = Futures.front(); + InFlightRequests.pop_front(); + Futures.pop_front(); + promise.SetValue(future.GetValue()); + } + if (NeedRecreation && Futures.empty() && !ReconnectionCallback) { + // We need recreation, but scheduled recreation hasn't start producer because of nonempty future list. + DEBUG_LOG("Recreating subproducer after all futures were processed", Settings.SourceId, ""); + RecreateProducer(TInstant::Now() + Settings.StartSessionTimeout); + } + } +} + +void TRetryingProducer::Destroy(const TString& description) { + TError error; + error.SetDescription(description); + error.SetCode(NErrorCode::ERROR); + Destroy(error); +} + +void TRetryingProducer::SubscribeDestroyed() { + NThreading::TPromise<void> promise = ProducersDestroyed; + auto handler = [promise](const auto&) mutable { + promise.SetValue(); + }; + if (Producer) { + WaitExceptionOrAll(DestroyedPromise.GetFuture(), Producer->Destroyed()) + .Subscribe(handler); + } else { + DestroyedPromise.GetFuture() + .Subscribe(handler); + } + + DestroyPQLibRef(); +} + +void TRetryingProducer::Destroy(const TError& error) { + if (Stopping) { + return; + } + Stopping = true; + SubscribeDestroyed(); + const bool started = StartFuture.Initialized(); + Producer = nullptr; + if (started) { + Y_VERIFY(Promises.size() == ResendRequests.size() + Requests.size() + InFlightRequests.size()); + ResendRequests.insert(ResendRequests.begin(), InFlightRequests.begin(), InFlightRequests.end()); + ResendRequests.insert(ResendRequests.end(), Requests.begin(), Requests.end()); + for (auto& v : ResendRequests) { + TWriteResponse resp; + *resp.MutableError() = error; + Promises.front().SetValue(TProducerCommitResponse(v.SeqNo, std::move(v.Data), std::move(resp))); + Promises.pop_front(); + } + Y_VERIFY(Promises.empty()); + } + if (StartPromise.Initialized() && !StartPromise.HasValue()) { + TWriteResponse resp; + *resp.MutableError() = error; + StartPromise.SetValue(TProducerCreateResponse(std::move(resp))); + } + IsDeadPromise.SetValue(error); +} + +NThreading::TFuture<void> TRetryingProducer::Destroyed() noexcept { + return ProducersDestroyed.GetFuture(); +} + +void TRetryingProducer::OnStartDeadline() { + if (!StartPromise.HasValue()) { + TError error; + error.SetDescription("Start timeout."); + error.SetCode(NErrorCode::CREATE_TIMEOUT); + Destroy(error); + } +} + +void TRetryingProducer::Cancel() { + Destroy(GetCancelReason()); +} + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h new file mode 100644 index 0000000000..f1242564b9 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer.h @@ -0,0 +1,75 @@ +#pragma once + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include "scheduler.h" +#include "internals.h" +#include "persqueue_p.h" +#include "producer.h" +#include "iproducer_p.h" +#include <library/cpp/threading/future/future.h> +#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> +#include <deque> + +namespace NPersQueue { + +class TRetryingProducer: public IProducerImpl, public std::enable_shared_from_this<TRetryingProducer> { +public: + using IProducerImpl::Write; + // If seqno == 0, we assume it to be autodefined + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, const TProducerSeqNo seqNo, TData data) noexcept override; + void Write(NThreading::TPromise<TProducerCommitResponse>& promise, TData data) noexcept override { + TRetryingProducer::Write(promise, 0, std::move(data)); + } + + TRetryingProducer(const TProducerSettings& settings, std::shared_ptr<void> destroyEventRef, TIntrusivePtr<TPQLibPrivate> pqLib, + TIntrusivePtr<ILogger> logger); + + NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline) noexcept override; + + NThreading::TFuture<TError> IsDead() noexcept override; + + NThreading::TFuture<void> Destroyed() noexcept override; + + ~TRetryingProducer() noexcept; + + + void RecreateProducer(TInstant deadline) noexcept; + void ProcessStart(const NThreading::TFuture<TProducerCreateResponse>&) noexcept; + void SendData() noexcept; + void ProcessFutures() noexcept; + void DoRecreate(TInstant deadline) noexcept; + void DelegateWriteAndSubscribe(TProducerSeqNo seqNo, TData&& data) noexcept; + TDuration UpdateReconnectionDelay(); + void ScheduleRecreation(); + void Destroy(const TError& error); + void Destroy(const TString& description); // the same but with Code=ERROR + void SubscribeDestroyed(); + void OnStartDeadline(); + void OnProducerDead(const TError& error); + + void Cancel() override; + +protected: + TProducerSettings Settings; + TIntrusivePtr<ILogger> Logger; + std::shared_ptr<IProducerImpl> Producer; + std::deque<TWriteData> Requests; + std::deque<TWriteData> ResendRequests; + std::deque<TWriteData> InFlightRequests; + std::deque<NThreading::TPromise<TProducerCommitResponse>> Promises; + std::deque<NThreading::TFuture<TProducerCommitResponse>> Futures; + NThreading::TFuture<TProducerCreateResponse> StartFuture; + NThreading::TPromise<TProducerCreateResponse> StartPromise; + NThreading::TPromise<TError> IsDeadPromise; + bool NeedRecreation; + bool Stopping; + ui32 ToProcess; + TDuration LastReconnectionDelay; + unsigned ReconnectionAttemptsDone; + TIntrusivePtr<TScheduler::TCallbackHandler> ReconnectionCallback; + NThreading::TPromise<void> ProducersDestroyed = NThreading::NewPromise<void>(); +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp new file mode 100644 index 0000000000..3fdb17d954 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/retrying_producer_ut.cpp @@ -0,0 +1,355 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h> + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + +namespace NPersQueue { + +Y_UNIT_TEST_SUITE(TRetryingProducerTest) { + TProducerSettings FakeSettings() { + TProducerSettings settings; + settings.ReconnectOnFailure = true; + settings.Server = TServerSetting{"localhost"}; + settings.Topic = "topic"; + settings.SourceId = "src"; + return settings; + } + + Y_UNIT_TEST(NotStartedProducerCanBeDestructed) { + // Test that producer doesn't hang on till shutdown + TPQLib lib; + TProducerSettings settings = FakeSettings(); + lib.CreateProducer(settings, {}, false); + } + + Y_UNIT_TEST(StartDeadlineExpires) { + if (std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED")) + return; + TPQLibSettings libSettings; + libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); + libSettings.DefaultLogger = new TCerrLogger(TLOG_DEBUG); + TPQLib lib(libSettings); + THolder<IProducer> producer; + { + TProducerSettings settings = FakeSettings(); + settings.ReconnectionDelay = TDuration::MilliSeconds(1); + settings.StartSessionTimeout = TDuration::MilliSeconds(10); + producer = lib.CreateProducer(settings, {}, false); + } + const TInstant beforeStart = TInstant::Now(); + auto future = producer->Start(TDuration::MilliSeconds(100)); + UNIT_ASSERT(future.GetValueSync().Response.HasError()); + UNIT_ASSERT_EQUAL_C(future.GetValueSync().Response.GetError().GetCode(), NErrorCode::CREATE_TIMEOUT, + "Error: " << future.GetValueSync().Response.GetError()); + const TInstant now = TInstant::Now(); + UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(100), now); + + DestroyAndWait(producer); + } + + Y_UNIT_TEST(StartMaxAttemptsExpire) { + TPQLibSettings libSettings; + libSettings.ChannelCreationTimeout = TDuration::MilliSeconds(1); + TPQLib lib(libSettings); + THolder<IProducer> producer; + { + TProducerSettings settings = FakeSettings(); + settings.ReconnectionDelay = TDuration::MilliSeconds(10); + settings.StartSessionTimeout = TDuration::MilliSeconds(10); + settings.MaxAttempts = 3; + producer = lib.CreateProducer(settings, {}, false); + } + const TInstant beforeStart = TInstant::Now(); + auto future = producer->Start(); + UNIT_ASSERT(future.GetValueSync().Response.HasError()); + const TInstant now = TInstant::Now(); + UNIT_ASSERT_C(now - beforeStart >= TDuration::MilliSeconds(30), now); + + DestroyAndWait(producer); + } + + void AssertWriteValid(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { + const TProducerCommitResponse& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kAck, "Msg: " << resp.Response); + } + + void AssertWriteFailed(const NThreading::TFuture<TProducerCommitResponse>& respFuture) { + const TProducerCommitResponse& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TWriteResponse::kError, "Msg: " << resp.Response); + } + + void AssertReadValid(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kData, "Msg: " << resp.Response); + } + + void AssertReadContinuous(const TReadResponse::TData::TMessageBatch& batch, ui64 startSeqNo) { + for (ui32 i = 0; i < batch.MessageSize(); ++i) { + ui64 actualSeqNo = batch.GetMessage(i).GetMeta().GetSeqNo(); + UNIT_ASSERT_EQUAL_C(actualSeqNo, startSeqNo + i, "Wrong seqNo: " << actualSeqNo << ", expected: " << startSeqNo + i); + } + } + + void AssertCommited(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kCommit, "Msg: " << resp.Response); + } + + void AssertReadFailed(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kError, "Msg: " << resp.Response); + } + + void AssertLock(const NThreading::TFuture<TConsumerMessage>& respFuture) { + const TConsumerMessage& resp = respFuture.GetValueSync(); + UNIT_ASSERT_EQUAL_C(resp.Response.GetResponseCase(), TReadResponse::kLock, "Msg: " << resp.Response); + } + + TProducerSettings MakeProducerSettings(const TTestServer& testServer) { + TProducerSettings producerSettings; + producerSettings.ReconnectOnFailure = true; + producerSettings.Topic = "topic1"; + producerSettings.SourceId = "123"; + producerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + producerSettings.Codec = ECodec::LZOP; + producerSettings.ReconnectionDelay = TDuration::MilliSeconds(10); + return producerSettings; + } + + TConsumerSettings MakeConsumerSettings(const TTestServer& testServer) { + TConsumerSettings consumerSettings; + consumerSettings.ClientId = "user"; + consumerSettings.Server = TServerSetting{"localhost", testServer.GrpcPort}; + consumerSettings.Topics.push_back("topic1"); + return consumerSettings; + } + + Y_UNIT_TEST(ReconnectsToServer) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + const size_t partitions = 10; + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + testServer.WaitInit("topic1"); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLib PQLib; + + auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); + producer->Start().Wait(); + TFuture<TError> isDead = producer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); + consumer->Start().Wait(); + + auto read1 = consumer->GetNextMessage(); + + // write first value + auto write1 = producer->Write(1, TString("blob1")); + AssertWriteValid(write1); + AssertReadValid(read1); + + // commit + consumer->Commit({1}); + auto commitAck = consumer->GetNextMessage(); + AssertCommited(commitAck); + + auto read2 = consumer->GetNextMessage(); + testServer.ShutdownServer(); + UNIT_ASSERT(!isDead.HasValue()); + + auto write2 = producer->Write(2, TString("blob2")); + + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + + AssertWriteValid(write2); + + AssertReadFailed(read2); + consumer->IsDead().Wait(); + + consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); + consumer->Start().Wait(); + + read2 = consumer->GetNextMessage(); + AssertReadValid(read2); + + UNIT_ASSERT(!isDead.HasValue()); + + DestroyAndWait(producer); + DestroyAndWait(consumer); + } + + static void DiesOnTooManyReconnectionAttempts(bool callWrite) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", 2); + testServer.WaitInit("topic1"); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLib PQLib; + + TProducerSettings settings = MakeProducerSettings(testServer); + settings.MaxAttempts = 3; + settings.ReconnectionDelay = TDuration::MilliSeconds(100); + auto producer = PQLib.CreateProducer(settings, logger, false); + producer->Start().Wait(); + TFuture<TError> isDead = producer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + // shutdown server + const TInstant beforeShutdown = TInstant::Now(); + testServer.ShutdownServer(); + + NThreading::TFuture<TProducerCommitResponse> write; + if (callWrite) { + write = producer->Write(TString("data")); + } + + isDead.Wait(); + const TInstant afterDead = TInstant::Now(); + // 3 attempts: 100ms, 200ms and 400ms + UNIT_ASSERT_C(afterDead - beforeShutdown >= TDuration::MilliSeconds(700), "real difference: " << (afterDead - beforeShutdown)); + + if (callWrite) { + AssertWriteFailed(write); + } + } + + Y_UNIT_TEST(WriteDataThatWasSentBeforeConnectToServer) { + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(false); + + testServer.AnnoyingClient->FullInit(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + const size_t partitions = 10; + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + + testServer.WaitInit("topic1"); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLib PQLib; + + auto producer = PQLib.CreateProducer(MakeProducerSettings(testServer), logger, false); + auto producerStarted = producer->Start(); + auto write1 = producer->Write(1, TString("a")); + producerStarted.Wait(); + TVector<TFuture<TProducerCommitResponse>> acks; + for (int i = 2; i <= 1000; ++i) { + acks.push_back(producer->Write(i, TString("b"))); + } + + AssertWriteValid(write1); + for (auto& ack: acks) { + ack.Wait(); + } + + auto consumer = PQLib.CreateConsumer(MakeConsumerSettings(testServer), logger, false); + consumer->Start().Wait(); + + ui64 readMessages = 0; + while (readMessages < 1000) { + auto read1 = consumer->GetNextMessage(); + AssertReadValid(read1); + auto& batch = read1.GetValueSync().Response.GetData().GetMessageBatch(0); + AssertReadContinuous(batch, readMessages + 1); + readMessages += batch.MessageSize(); + } + + DestroyAndWait(producer); + DestroyAndWait(consumer); + } + + Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithoutWrite) { + // Check that we reconnect even without explicit write errors + DiesOnTooManyReconnectionAttempts(false); + } + + Y_UNIT_TEST(DiesOnTooManyReconnectionAttemptsWithWrite) { + DiesOnTooManyReconnectionAttempts(true); + } + + Y_UNIT_TEST(CancelsOperationsAfterPQLibDeath) { + return; // Test is ignored. FIX: KIKIMR-7886 + TTestServer testServer(false); + testServer.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::MilliSeconds(10)); + testServer.StartServer(); + + const size_t partitions = 1; + testServer.AnnoyingClient->InitRoot(); + testServer.AnnoyingClient->InitDCs(!GrpcV1EnabledByDefault() ? DEFAULT_CLUSTERS_LIST : CLUSTERS_LIST_ONE_DC); + testServer.AnnoyingClient->CreateTopicNoLegacy("rt3.dc1--topic1", partitions); + testServer.AnnoyingClient->InitSourceIds(); + + testServer.WaitInit("topic1"); + + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLibSettings pqLibSettings; + pqLibSettings.DefaultLogger = logger; + THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); + + auto producer = PQLib->CreateProducer(MakeProducerSettings(testServer), logger, false); + UNIT_ASSERT(!producer->Start().GetValueSync().Response.HasError()); + TFuture<TError> isDead = producer->IsDead(); + UNIT_ASSERT(!isDead.HasValue()); + + testServer.ShutdownServer(); + UNIT_ASSERT(!isDead.HasValue()); + + auto write1 = producer->Write(1, TString("blob1")); + auto write2 = producer->Write(2, TString("blob2")); + + UNIT_ASSERT(!write1.HasValue()); + UNIT_ASSERT(!write2.HasValue()); + + PQLib = nullptr; + + UNIT_ASSERT(write1.HasValue()); + UNIT_ASSERT(write2.HasValue()); + + UNIT_ASSERT(write1.GetValue().Response.HasError()); + UNIT_ASSERT(write2.GetValue().Response.HasError()); + + auto write3 = producer->Write(3, TString("blob3")); + UNIT_ASSERT(write3.HasValue()); + UNIT_ASSERT(write3.GetValue().Response.HasError()); + } + + Y_UNIT_TEST(CancelsStartAfterPQLibDeath) { + TIntrusivePtr<TCerrLogger> logger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + TPQLibSettings pqLibSettings; + pqLibSettings.DefaultLogger = logger; + THolder<TPQLib> PQLib = MakeHolder<TPQLib>(pqLibSettings); + + auto producer = PQLib->CreateProducer(FakeSettings(), logger, false); + auto start = producer->Start(); + UNIT_ASSERT(!start.HasValue()); + + PQLib = nullptr; + + UNIT_ASSERT(start.HasValue()); + UNIT_ASSERT(start.GetValue().Response.HasError()); + + auto dead = producer->IsDead(); + UNIT_ASSERT(dead.HasValue()); + dead.GetValueSync(); + + auto write = producer->Write(1, TString("blob1")); + UNIT_ASSERT(write.HasValue()); + UNIT_ASSERT(write.GetValueSync().Response.HasError()); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp new file mode 100644 index 0000000000..ba6c4ebb40 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.cpp @@ -0,0 +1,96 @@ +#include "persqueue_p.h" +#include "scheduler.h" + +#include <util/system/thread.h> + +namespace NPersQueue { +void TScheduler::TCallbackHandler::Execute() { + auto guard = Guard(Lock); + if (Callback) { + Y_VERIFY(PQLib->GetQueuePool().GetQueue(QueueTag).AddFunc(std::move(Callback))); + Callback = nullptr; + } +} + +void TScheduler::TCallbackHandler::TryCancel() { + auto guard = Guard(Lock); + if (Callback) { + Callback = nullptr; + } +} + +bool TScheduler::TCallbackHandlersCompare::operator()(const TIntrusivePtr<TScheduler::TCallbackHandler>& h1, const TIntrusivePtr<TScheduler::TCallbackHandler>& h2) const { + return h1->Time > h2->Time; +} + +TScheduler::TScheduler(TPQLibPrivate* pqLib) + : Shutdown(false) + , PQLib(pqLib) +{ + Thread = SystemThreadFactory()->Run([this] { + this->SchedulerThread(); + }); +} + +TScheduler::~TScheduler() { + ShutdownAndWait(); +} + +void TScheduler::AddToSchedule(TIntrusivePtr<TCallbackHandler> handler) { + { + auto guard = Guard(Lock); + Callbacks.push(std::move(handler)); + } + Event.Signal(); +} + +void TScheduler::ShutdownAndWait() { + AtomicSet(Shutdown, true); + Event.Signal(); + Thread->Join(); +} + +void TScheduler::SchedulerThread() { + TThread::SetCurrentThreadName("pqlib_scheduler"); + while (!AtomicGet(Shutdown)) { + TInstant deadline = TInstant::Max(); + std::vector<TIntrusivePtr<TCallbackHandler>> callbacks; + { + // define next deadline and get expired callbacks + auto guard = Guard(Lock); + const TInstant now = TInstant::Now(); + while (!Callbacks.empty()) { + const auto& top = Callbacks.top(); + if (top->Time <= now) { + callbacks.push_back(top); + Callbacks.pop(); + } else { + deadline = top->Time; + break; + } + } + } + + // execute callbacks + bool shutdown = false; + for (auto& callback : callbacks) { + if (shutdown) { + callback->TryCancel(); + } else { + callback->Execute(); + } + shutdown = shutdown || AtomicGet(Shutdown); + } + + if (!shutdown) { + Event.WaitD(deadline); + } + } + + // cancel all callbacks and clear data + while (!Callbacks.empty()) { + Callbacks.top()->TryCancel(); + Callbacks.pop(); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h new file mode 100644 index 0000000000..c90ab87795 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler.h @@ -0,0 +1,88 @@ +#pragma once +#include <util/datetime/base.h> +#include <util/generic/ptr.h> +#include <util/system/event.h> +#include <util/system/spinlock.h> +#include <util/thread/factory.h> + +#include <functional> +#include <queue> +#include <utility> + +namespace NPersQueue { + +class TPQLibPrivate; + +class TScheduler { + struct TCallbackHandlersCompare; +public: + using TCallback = std::function<void()>; + + class TCallbackHandler: public TAtomicRefCount<TCallbackHandler> { + friend class TScheduler; + friend struct TCallbackHandlersCompare; + + template <class TFunc> + TCallbackHandler(TInstant time, TFunc&& func, TPQLibPrivate* pqLib, const void* queueTag) + : Time(time) + , Callback(std::forward<TFunc>(func)) + , PQLib(pqLib) + , QueueTag(queueTag) + { + } + + void Execute(); + + public: + // Cancels execution of callback. + // If callback is already canceled or executes (executed), does nothing. + // Posteffect: callback is guaranteed to be destroyed after this call. + void TryCancel(); + + private: + TInstant Time; + TCallback Callback; + TAdaptiveLock Lock; + TPQLibPrivate* PQLib; + const void* QueueTag; + }; + +public: + // Starts a scheduler thread. + explicit TScheduler(TPQLibPrivate* pqLib); + + // Stops a scheduler thread. + ~TScheduler(); + + // Schedules a new callback to be executed + template <class TFunc> + TIntrusivePtr<TCallbackHandler> Schedule(TInstant time, const void* queueTag, TFunc&& func) { + TIntrusivePtr<TCallbackHandler> handler(new TCallbackHandler(time, std::forward<TFunc>(func), PQLib, queueTag)); + AddToSchedule(handler); + return handler; + } + + template <class TFunc> + TIntrusivePtr<TCallbackHandler> Schedule(TDuration delta, const void* queueTag, TFunc&& func) { + return Schedule(TInstant::Now() + delta, queueTag, std::forward<TFunc>(func)); + } + +private: + void AddToSchedule(TIntrusivePtr<TCallbackHandler> handler); + void ShutdownAndWait(); + void SchedulerThread(); + +private: + struct TCallbackHandlersCompare { + bool operator()(const TIntrusivePtr<TCallbackHandler>& h1, const TIntrusivePtr<TCallbackHandler>& h2) const; + }; + + using TCallbackQueue = std::priority_queue<TIntrusivePtr<TCallbackHandler>, std::vector<TIntrusivePtr<TCallbackHandler>>, TCallbackHandlersCompare>; + TAdaptiveLock Lock; + TAutoEvent Event; + TCallbackQueue Callbacks; + TAtomic Shutdown; + THolder<IThreadFactory::IThread> Thread; + TPQLibPrivate* PQLib; +}; +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp new file mode 100644 index 0000000000..83ce25c6b4 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/scheduler_ut.cpp @@ -0,0 +1,75 @@ +#include "scheduler.h" +#include "persqueue_p.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/event.h> + +namespace NPersQueue { +const void* Tag = (const void*)42; +Y_UNIT_TEST_SUITE(TSchedulerTest) { + Y_UNIT_TEST(EmptySchedulerCanDestruct) { + // Test that scheduler doesn't hang on till shutdown + TScheduler scheduler(nullptr); + } + + Y_UNIT_TEST(ExecutesInProperOrder) { + TSystemEvent ev; + bool callback1Executed = false; + TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({})); + TScheduler scheduler(lib.Get()); + scheduler.Schedule(TDuration::MilliSeconds(100), Tag, [&callback1Executed, &ev] { + UNIT_ASSERT(callback1Executed); + ev.Signal(); + }); + scheduler.Schedule(TDuration::MilliSeconds(50), Tag, [&callback1Executed] { + callback1Executed = true; + }); + ev.Wait(); + } + + Y_UNIT_TEST(CancelsAndClearsData) { + TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({})); + TScheduler scheduler(lib.Get()); + std::shared_ptr<TString> ptr(new TString()); + bool callbackExecuted = false; + auto h = scheduler.Schedule(TDuration::Seconds(50), Tag, [&callbackExecuted, ptr] { + callbackExecuted = true; + }); + UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 2); + h->TryCancel(); + UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1); + UNIT_ASSERT(!callbackExecuted); + Sleep(TDuration::MilliSeconds(51)); + UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1); + UNIT_ASSERT(!callbackExecuted); + } + + Y_UNIT_TEST(ExitsThreadImmediately) { + TIntrusivePtr<TPQLibPrivate> lib(new TPQLibPrivate({})); + std::shared_ptr<TString> ptr(new TString()); + bool callback1Executed = false; + bool callback2Executed = false; + TSystemEvent ev, ev2; + auto now = TInstant::Now(); + lib->GetScheduler().Schedule(TDuration::Seconds(500), Tag, [&callback1Executed, ptr] { + callback1Executed = true; + }); + lib->GetScheduler().Schedule(now, Tag, [&callback2Executed, &ev, &ev2, ptr] { + callback2Executed = true; + ev2.Wait(); + ev.Signal(); + }); + UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 3); + ev2.Signal(); + + // kill scheduler + ev.Wait(); + lib.Reset(); + + UNIT_ASSERT_VALUES_EQUAL(ptr.use_count(), 1); + UNIT_ASSERT(!callback1Executed); + UNIT_ASSERT(callback2Executed); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp new file mode 100644 index 0000000000..4fc6a1c7b4 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types.cpp @@ -0,0 +1,86 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> + +#include <library/cpp/streams/lzop/lzop.h> +#include <library/cpp/streams/zstd/zstd.h> + +#include <util/generic/store_policy.h> +#include <util/generic/utility.h> +#include <util/stream/str.h> +#include <util/stream/zlib.h> + +namespace NPersQueue { + +const TString& TServerSetting::GetRootDatabase() { + static const TString RootDatabase = "/Root"; + return RootDatabase; +} + +class TZLibToStringCompressor: private TEmbedPolicy<TStringOutput>, public TZLibCompress { +public: + TZLibToStringCompressor(TString& dst, ZLib::StreamType type, size_t quality) + : TEmbedPolicy<TStringOutput>(dst) + , TZLibCompress(TEmbedPolicy::Ptr(), type, quality) + { + } +}; + +class TLzopToStringCompressor: private TEmbedPolicy<TStringOutput>, public TLzopCompress { +public: + TLzopToStringCompressor(TString& dst) + : TEmbedPolicy<TStringOutput>(dst) + , TLzopCompress(TEmbedPolicy::Ptr()) + { + } +}; + +class TZstdToStringCompressor: private TEmbedPolicy<TStringOutput>, public TZstdCompress { +public: + TZstdToStringCompressor(TString& dst, int quality) + : TEmbedPolicy<TStringOutput>(dst) + , TZstdCompress(TEmbedPolicy::Ptr(), quality) + { + } +}; + + +TData TData::Encode(TData source, ECodec defaultCodec, int quality) { + Y_VERIFY(!source.Empty()); + TData data = std::move(source); + if (data.IsEncoded()) { + return data; + } + Y_VERIFY(defaultCodec != ECodec::RAW && defaultCodec != ECodec::DEFAULT); + if (data.Codec == ECodec::DEFAULT) { + data.Codec = defaultCodec; + } + THolder<IOutputStream> coder = CreateCoder(data.Codec, data, quality); + coder->Write(data.SourceData); + coder->Finish(); // &data.EncodedData may be already invalid on coder destruction + return data; +} + +TData TData::MakeRawIfNotEncoded(TData source) { + Y_VERIFY(!source.Empty()); + TData data = std::move(source); + if (!data.IsEncoded()) { + data.EncodedData = data.SourceData; + data.Codec = ECodec::RAW; + } + return data; +} + +THolder<IOutputStream> TData::CreateCoder(ECodec codec, TData& result, int quality) { + switch (codec) { + case ECodec::GZIP: + return MakeHolder<TZLibToStringCompressor>(result.EncodedData, ZLib::GZip, quality >= 0 ? quality : 6); + case ECodec::LZOP: + return MakeHolder<TLzopToStringCompressor>(result.EncodedData); + case ECodec::ZSTD: + return MakeHolder<TZstdToStringCompressor>(result.EncodedData, quality); + default: + Y_FAIL("NOT IMPLEMENTED CODEC TYPE"); + } +} + + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp new file mode 100644 index 0000000000..06c4b28454 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/types_ut.cpp @@ -0,0 +1,88 @@ +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/string/builder.h> + +namespace NPersQueue { +Y_UNIT_TEST_SUITE(TDataTest) { + ECodec Codecs[] = { + ECodec::LZOP, + ECodec::GZIP, + }; + + TString DebugString(const TData& data) { + return TStringBuilder() << "{ Ts: " << data.GetTimestamp().GetValue() << ", SrcData: \"" << data.GetSourceData() + << "\", Encoded: \"" << (data.IsEncoded() ? data.GetEncodedData() : TString()) + << "\", Codec: " << data.GetCodecType() << " }"; + } + + Y_UNIT_TEST(RawDataIsEncoded) { + auto now = TInstant::Now(); + TData data = TData::Raw("trololo", now); + TData otherRawData = TData("trololo", ECodec::RAW, now); + UNIT_ASSERT_C(data.IsEncoded(), "data: " << DebugString(data)); + UNIT_ASSERT_C(!data.Empty(), "data: " << DebugString(data)); + UNIT_ASSERT_EQUAL_C(data.GetCodecType(), ECodec::RAW, "data: " << DebugString(data)); + UNIT_ASSERT_EQUAL_C(data, otherRawData, "data: " << DebugString(data) << ", other: " << DebugString(otherRawData)); + } + + Y_UNIT_TEST(EncodedDataIsEncoded) { + for (ECodec codec : Codecs) { + TData data = TData::Encoded("trololo", codec); + UNIT_ASSERT(data.IsEncoded()); + UNIT_ASSERT(!data.Empty()); + UNIT_ASSERT_EQUAL(data.GetCodecType(), codec); + } + } + + Y_UNIT_TEST(ModifiesState) { + for (ECodec codec : Codecs) { + for (ECodec defaultCodec : Codecs) { + TData data("trololo", codec); + UNIT_ASSERT(!data.IsEncoded()); + UNIT_ASSERT(!data.Empty()); + + for (size_t i = 0; i < 2; ++i) { + data = TData::Encode(std::move(data), defaultCodec, 2); // encode twice is OK + UNIT_ASSERT(data.IsEncoded()); + UNIT_ASSERT(!data.Empty()); + UNIT_ASSERT(!data.GetEncodedData().empty()); + UNIT_ASSERT_EQUAL(data.GetCodecType(), codec); + } + } + } + } + + Y_UNIT_TEST(HandlesDefaultCodec) { + for (ECodec defaultCodec : Codecs) { + TData data = TString("trololo"); + UNIT_ASSERT(!data.IsEncoded()); + UNIT_ASSERT(!data.Empty()); + + data = TData::Encode(data, defaultCodec, -1); + UNIT_ASSERT(data.IsEncoded()); + UNIT_ASSERT(!data.Empty()); + UNIT_ASSERT_STRINGS_EQUAL(data.GetSourceData(), "trololo"); + UNIT_ASSERT(!data.GetEncodedData().empty()); + UNIT_ASSERT_EQUAL(data.GetCodecType(), defaultCodec); + } + } + + Y_UNIT_TEST(MakesRaw) { + TData data = TString("trololo"); + UNIT_ASSERT(!data.IsEncoded()); + data = TData::MakeRawIfNotEncoded(data); + UNIT_ASSERT(data.IsEncoded()); + UNIT_ASSERT_EQUAL(data.GetCodecType(), ECodec::RAW); + } + + Y_UNIT_TEST(DoesNotMakeRaw) { + TData data = TData::Encoded("trololo", ECodec::GZIP); + UNIT_ASSERT(data.IsEncoded()); + data = TData::MakeRawIfNotEncoded(data); + UNIT_ASSERT(data.IsEncoded()); + UNIT_ASSERT_EQUAL(data.GetCodecType(), ECodec::GZIP); + } +} +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp new file mode 100644 index 0000000000..a7d65ddc88 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.cpp @@ -0,0 +1,24 @@ +#include "validate_grpc_metadata.h" +#include <contrib/libs/grpc/include/grpc/grpc.h> +#include <util/generic/string.h> +#include <util/string/escape.h> +#include <util/string/builder.h> + +namespace NGrpc { + bool ValidateHeaderIsLegal(const TString& key, const TString& value, TString& error) { + error.clear(); + grpc_slice keySlice = grpc_slice_from_static_buffer(key.c_str(), key.size()); + int ok = grpc_header_key_is_legal(keySlice); + if (!ok) { + error = TStringBuilder() << "gRPC metadata header key is illegal: \"" << EscapeC(key) << "\""; + return false; + } + grpc_slice valueSlice = grpc_slice_from_static_buffer(value.c_str(), value.size()); + ok = grpc_is_binary_header(keySlice) || grpc_header_nonbin_value_is_legal(valueSlice); + if (!ok) { + error = TStringBuilder() << "gRPC metadata header value with key \"" << key << "\" is illegal: \"" << EscapeC(value) << "\""; + return false; + } + return true; + } +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h new file mode 100644 index 0000000000..43fd82cef0 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata.h @@ -0,0 +1,12 @@ +#pragma once +#include <util/generic/fwd.h> + +namespace NGrpc { + // Validates gRPC metadata header key and value. Returns 'true' if validations fails, otherwise returns 'false' and sets 'error' value. + // For more information see 'Custom-Metadata' section in gRPC over HTTP2 (https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md). + // Note that in case of authentication data validation 'error' may contain sensitive information. + bool ValidateHeaderIsLegal(const TString& key, const TString& value, TString& error); + + // Creates hexadecimal representation of 's' in format '{0x01, 0x02, 0x03}' + TString ToHexString(const TString& s); +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp new file mode 100644 index 0000000000..bbda57f538 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/validate_grpc_metadata_ut.cpp @@ -0,0 +1,46 @@ +#include "validate_grpc_metadata.h" +#include <library/cpp/testing/unittest/registar.h> + +using namespace NGrpc; + +Y_UNIT_TEST_SUITE(NValidateGrpcMetadata) { + Y_UNIT_TEST(ValidateHeaderIsLegal) { + TString error = "error"; + UNIT_ASSERT_C(ValidateHeaderIsLegal("key", "value", error), error); + // Assert 'error' is cleared + UNIT_ASSERT_C(error.empty(), error); + + // Valid character values upper and lower bounds + UNIT_ASSERT_C(ValidateHeaderIsLegal("\x30\x39\x61\x7A_-.", "\x20\x7E", error), error); + UNIT_ASSERT_C(error.empty(), error); + + TString null = " "; + null[0] = '\0'; + UNIT_ASSERT(!ValidateHeaderIsLegal("key", null, error)); + UNIT_ASSERT_C(error.Contains("\\0"), error); + Cerr << "Error is '" << error << "'" << Endl; + + // Simple escape sequences + UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x0A\t", error)); + UNIT_ASSERT_C(error.Contains("\\n\\t"), error); + Cerr << "Error is '" << error << "'" << Endl; + + UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x1F", error)); + UNIT_ASSERT_C(error.Contains("\\x1F"), error); + Cerr << "Error is '" << error << "'" << Endl; + + UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\x7F", error)); + UNIT_ASSERT_C(error.Contains("\\x7F"), error); + Cerr << "Error is '" << error << "'" << Endl; + + // Octal character + UNIT_ASSERT(!ValidateHeaderIsLegal("key", "value\177", error)); + UNIT_ASSERT_C(error.Contains("\\x7F"), error); + Cerr << "Error is '" << error << "'" << Endl; + + // Invalid header key + UNIT_ASSERT(!ValidateHeaderIsLegal("key\n", "value", error)); + UNIT_ASSERT_C(error.Contains("\\n"), error); + Cerr << "Error is '" << error << "'" << Endl; + } +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp new file mode 100644 index 0000000000..b679eda9f4 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.cpp @@ -0,0 +1,561 @@ +#include "ydb_sdk_consumer.h" +#include "persqueue_p.h" + +#include <ydb/library/persqueue/topic_parser_public/topic_parser.h> + +#include <ydb/public/sdk/cpp/client/ydb_persqueue_core/impl/common.h> +#include <util/generic/is_in.h> +#include <util/string/builder.h> + +namespace NPersQueue { + +static TError MakeError(const TString& description, NErrorCode::EErrorCode code) { + TError error; + error.SetDescription(description); + error.SetCode(code); + return error; +} + +static TString MakeLegacyTopicName(const NYdb::NPersQueue::TPartitionStream::TPtr& partitionStream) { + // rt3.man--account--topic + return BuildFullTopicName(partitionStream->GetTopicPath(), partitionStream->GetCluster()); +} + +static NErrorCode::EErrorCode ToErrorCode(NYdb::EStatus status) { + switch (status) { + case NYdb::EStatus::STATUS_UNDEFINED: + return NErrorCode::ERROR; + case NYdb::EStatus::SUCCESS: + return NErrorCode::OK; + case NYdb::EStatus::BAD_REQUEST: + return NErrorCode::BAD_REQUEST; + case NYdb::EStatus::UNAUTHORIZED: + return NErrorCode::ACCESS_DENIED; + case NYdb::EStatus::INTERNAL_ERROR: + return NErrorCode::ERROR; + case NYdb::EStatus::ABORTED: + [[fallthrough]]; + case NYdb::EStatus::UNAVAILABLE: + return NErrorCode::ERROR; + case NYdb::EStatus::OVERLOADED: + return NErrorCode::OVERLOAD; + case NYdb::EStatus::SCHEME_ERROR: + return NErrorCode::UNKNOWN_TOPIC; + case NYdb::EStatus::GENERIC_ERROR: + [[fallthrough]]; + case NYdb::EStatus::TIMEOUT: + [[fallthrough]]; + case NYdb::EStatus::BAD_SESSION: + [[fallthrough]]; + case NYdb::EStatus::PRECONDITION_FAILED: + [[fallthrough]]; + case NYdb::EStatus::ALREADY_EXISTS: + [[fallthrough]]; + case NYdb::EStatus::NOT_FOUND: + [[fallthrough]]; + case NYdb::EStatus::SESSION_EXPIRED: + [[fallthrough]]; + case NYdb::EStatus::CANCELLED: + [[fallthrough]]; + case NYdb::EStatus::UNDETERMINED: + [[fallthrough]]; + case NYdb::EStatus::UNSUPPORTED: + [[fallthrough]]; + case NYdb::EStatus::SESSION_BUSY: + [[fallthrough]]; + case NYdb::EStatus::TRANSPORT_UNAVAILABLE: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_RESOURCE_EXHAUSTED: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_DEADLINE_EXCEEDED: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_INTERNAL_ERROR: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_CANCELLED: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_UNAUTHENTICATED: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_CALL_UNIMPLEMENTED: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_OUT_OF_RANGE: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_DISCOVERY_FAILED: + [[fallthrough]]; + case NYdb::EStatus::CLIENT_LIMITS_REACHED: + return NErrorCode::ERROR; + } +} + +TYdbSdkCompatibilityConsumer::TYdbSdkCompatibilityConsumer(const TConsumerSettings& settings, + std::shared_ptr<void> destroyEventRef, + TIntrusivePtr<TPQLibPrivate> pqLib, + TIntrusivePtr<ILogger> logger, + NYdb::NPersQueue::TPersQueueClient& client) + : IConsumerImpl(std::move(destroyEventRef), std::move(pqLib)) + , Settings(settings) + , Logger(std::move(logger)) + , Client(client) +{ +} + +TYdbSdkCompatibilityConsumer::~TYdbSdkCompatibilityConsumer() { + Destroy("Destructor called"); +} + +void TYdbSdkCompatibilityConsumer::Init() { + YdbSdkSettings = MakeReadSessionSettings(); // weak_from_this() is used here. +} + +static const THashSet<TString> FederationClusters = { + "iva", + "man", + "myt", + "sas", + "vla", +}; + +static TString FindClusterFromEndpoint(const TString& endpoint) { + size_t pos = endpoint.find(".logbroker.yandex.net"); + if (pos == TString::npos) { + pos = endpoint.find(".logbroker-prestable.yandex.net"); + if (pos == TString::npos) { + return {}; // something strange or cross dc cloud cluster. + } + } + TString prefix = endpoint.substr(0, pos); + if (IsIn(FederationClusters, prefix)) { + return prefix; + } + return {}; // no cluster in cross dc federation. +} + +NYdb::NPersQueue::TReadSessionSettings TYdbSdkCompatibilityConsumer::MakeReadSessionSettings() { + NYdb::NPersQueue::TReadSessionSettings settings; + for (const TString& topic : Settings.Topics) { + settings.AppendTopics(topic); + } + settings.ConsumerName(Settings.ClientId); + settings.MaxMemoryUsageBytes(Settings.MaxMemoryUsage); + if (Settings.MaxTimeLagMs) { + settings.MaxTimeLag(TDuration::MilliSeconds(Settings.MaxTimeLagMs)); + } + if (Settings.ReadTimestampMs) { + settings.StartingMessageTimestamp(TInstant::MilliSeconds(Settings.ReadTimestampMs)); + } + if (Settings.PartitionGroups.size()) { + Y_ENSURE(settings.Topics_.size() == 1); + for (ui64 group : Settings.PartitionGroups) { + settings.Topics_[0].AppendPartitionGroupIds(group); + } + } + if (Settings.ReconnectOnFailure || Settings.ReadFromAllClusterSources) { // ReadFromAllClusterSources implies ReconnectOnFailure and MaxAttempts == inf. + size_t maxRetries = Settings.MaxAttempts; + if (Settings.ReadFromAllClusterSources) { + // Compatibility. + maxRetries = std::numeric_limits<size_t>::max(); + } + + if (Settings.UseV2RetryPolicyInCompatMode) { + settings.RetryPolicy( + NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy( + Settings.ReconnectionDelay, + Settings.ReconnectionDelay, + Settings.MaxReconnectionDelay, + maxRetries, + TDuration::Max(), + 2.0, + NYdb::NPersQueue::GetRetryErrorClassV2 + )); + } else { + settings.RetryPolicy( + NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy( + Settings.ReconnectionDelay, + Settings.ReconnectionDelay, + Settings.MaxReconnectionDelay, + maxRetries + )); + } + } else { + settings.RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()); + } + if (Settings.ReadFromAllClusterSources) { + settings.ReadAll(); + } else { + if (TString cluster = FindClusterFromEndpoint(Settings.Server.Address)) { + if (Settings.ReadMirroredPartitions) { + settings.ReadMirrored(cluster); + } else { + settings.ReadOriginal({ cluster }); + } + } else { + settings.ReadAll(); + } + } + if (Settings.DisableCDS) { + settings.DisableClusterDiscovery(true); + } + + settings.EventHandlers_.HandlersExecutor(NYdb::NPersQueue::CreateThreadPoolExecutorAdapter(PQLib->GetQueuePool().GetQueuePtr(this))); + { + auto weakThis = weak_from_this(); + settings.EventHandlers_.SessionClosedHandler( + [weakThis](const NYdb::NPersQueue::TSessionClosedEvent& event) { + if (auto sharedThis = weakThis.lock()) { + const TString description = event.GetIssues().ToString(); + sharedThis->Destroy(description, ToErrorCode(event.GetStatus())); + } + } + ); + } + + return settings; +} + +NThreading::TFuture<TConsumerCreateResponse> TYdbSdkCompatibilityConsumer::Start(TInstant) noexcept { + ReadSession = Client.CreateReadSession(YdbSdkSettings); + SessionId = ReadSession->GetSessionId(); + DEBUG_LOG("Create read session", "", SessionId); + SubscribeToNextEvent(); + TReadResponse resp; + resp.MutableInit(); + return NThreading::MakeFuture<TConsumerCreateResponse>(TConsumerCreateResponse(std::move(resp))); +} + +NThreading::TFuture<TError> TYdbSdkCompatibilityConsumer::IsDead() noexcept { + return DeadPromise.GetFuture(); +} + +void TYdbSdkCompatibilityConsumer::Destroy(const TError& description) { + if (DeadPromise.HasValue()) { + return; + } + + WARN_LOG("Destroying consumer: " << description, "", SessionId); + + while (!Requests.empty()) { + NPersQueue::TReadResponse resp; + *resp.MutableError() = description; + Requests.front().SetValue(TConsumerMessage(std::move(resp))); + Requests.pop(); + } + + if (ReadSession) { + ReadSession->Close(TDuration::Zero()); + ReadSession = nullptr; + } + + DeadPromise.SetValue(description); + + DestroyPQLibRef(); +} + +void TYdbSdkCompatibilityConsumer::Destroy(const TString& description, NErrorCode::EErrorCode code) { + Destroy(MakeError(description, code)); +} + +void TYdbSdkCompatibilityConsumer::Cancel() { + Destroy(GetCancelReason()); +} + +void TYdbSdkCompatibilityConsumer::GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept { + if (DeadPromise.HasValue()) { + NPersQueue::TReadResponse resp; + *resp.MutableError() = DeadPromise.GetFuture().GetValue(); + promise.SetValue(TConsumerMessage(std::move(resp))); + return; + } + Requests.push(promise); + AnswerToRequests(); + SubscribeToNextEvent(); +} + +void TYdbSdkCompatibilityConsumer::Commit(const TVector<ui64>& cookies) noexcept { + NPersQueue::TReadResponse resp; + for (ui64 cookie : cookies) { + if (cookie >= NextCookie) { + Destroy(TStringBuilder() << "Wrong cookie " << cookie, NErrorCode::WRONG_COOKIE); + return; + } + auto offsetsIt = CookieToOffsets.find(cookie); + if (offsetsIt != CookieToOffsets.end()) { + offsetsIt->second.first.Commit(); + CookiesRequestedToCommit[offsetsIt->second.second].emplace(cookie); + CookieToOffsets.erase(offsetsIt); + } else { + resp.MutableCommit()->AddCookie(cookie); + } + } + if (resp.HasCommit()) { + AddResponse(std::move(resp)); + } +} + +void TYdbSdkCompatibilityConsumer::RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept { + Y_UNUSED(topic); + Y_UNUSED(partition); + auto partitionStreamIt = CurrentPartitionStreams.find(generation); + if (partitionStreamIt != CurrentPartitionStreams.end()) { + partitionStreamIt->second->RequestStatus(); + } +} + +void TYdbSdkCompatibilityConsumer::AnswerToRequests() { + TVector<NYdb::NPersQueue::TReadSessionEvent::TEvent> events; + do { + events = ReadSession->GetEvents(); + for (NYdb::NPersQueue::TReadSessionEvent::TEvent& event : events) { + if (ReadSession) { + std::visit([this](auto&& ev) { return HandleEvent(std::move(ev)); }, event); + } + } + } while (!events.empty() && ReadSession); + + while (!Requests.empty() && !Responses.empty()) { + Requests.front().SetValue(std::move(Responses.front())); + Requests.pop(); + Responses.pop(); + } +} + +void TYdbSdkCompatibilityConsumer::SubscribeToNextEvent() { + if (!SubscribedToNextEvent && !Requests.empty() && ReadSession) { + SubscribedToNextEvent = true; + auto weakThis = weak_from_this(); + auto future = ReadSession->WaitEvent(); + Cerr << "SUBSCRIBING " << ReadSession->GetSessionId() << " future" << future.StateId()->Value() << "\n"; + PQLib->Subscribe(future, this, [weakThis](const NThreading::TFuture<void>&) { + if (auto sharedThis = weakThis.lock()) { + sharedThis->OnReadSessionEvent(); + } + }); + } +} + +void TYdbSdkCompatibilityConsumer::OnReadSessionEvent() { + if (DeadPromise.HasValue()) { + return; + } + + SubscribedToNextEvent = false; + AnswerToRequests(); + SubscribeToNextEvent(); +} + +void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent&& event) { + if (Settings.UseLockSession) { + auto& offsetSet = PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()]; + // Messages could contain holes in offset, but later commit ack will tell us right border. + // So we can easily insert the whole interval with holes included. + // It will be removed from set by specifying proper right border. + offsetSet.InsertInterval(event.GetMessages().front().GetOffset(), event.GetMessages().back().GetOffset() + 1); + } + const ui64 cookie = NextCookie++; + const auto& partitionStream = event.GetPartitionStream(); + auto& offsets = CookieToOffsets[cookie]; + offsets.first.Add(event); + offsets.second = partitionStream->GetPartitionStreamId(); + PartitionStreamToCookies[partitionStream->GetPartitionStreamId()].emplace(cookie); + NPersQueue::TReadResponse resp; + auto& dataResp = *resp.MutableData(); + dataResp.SetCookie(cookie); + auto& batchResp = *dataResp.AddMessageBatch(); + batchResp.SetTopic(MakeLegacyTopicName(partitionStream)); + batchResp.SetPartition(partitionStream->GetPartitionId()); + ui64 maxOffset = 0; + for (auto&& msg : event.GetMessages()) { + auto& msgResp = *batchResp.AddMessage(); + msgResp.SetOffset(msg.GetOffset()); + maxOffset = Max(maxOffset, msg.GetOffset()); + msgResp.SetData(msg.GetData()); + auto& metaResp = *msgResp.MutableMeta(); + metaResp.SetSourceId(msg.GetMessageGroupId()); + metaResp.SetSeqNo(msg.GetSeqNo()); + metaResp.SetCreateTimeMs(msg.GetCreateTime().MilliSeconds()); + metaResp.SetWriteTimeMs(msg.GetWriteTime().MilliSeconds()); + metaResp.SetIp(msg.GetIp()); + for (auto&& [k, v] : msg.GetMeta()->Fields) { + auto& kvResp = *metaResp.MutableExtraFields()->AddItems(); + kvResp.SetKey(k); + kvResp.SetValue(v); + } + } + MaxOffsetToCookie[std::make_pair(partitionStream->GetPartitionStreamId(), maxOffset)] = cookie; + AddResponse(std::move(resp)); +} +void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&& event) { + const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); + if (Settings.UseLockSession) { + auto& offsetSet = PartitionStreamToUncommittedOffsets[partitionStreamId]; + if (offsetSet.EraseInterval(0, event.GetCommittedOffset())) { // Remove some offsets. + if (offsetSet.Empty()) { // No offsets left. + auto unconfirmedDestroyIt = UnconfirmedDestroys.find(partitionStreamId); + if (unconfirmedDestroyIt != UnconfirmedDestroys.end()) { + // Confirm and forget about this partition stream. + unconfirmedDestroyIt->second.Confirm(); + UnconfirmedDestroys.erase(unconfirmedDestroyIt); + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + } + } + } + } + const auto offsetPair = std::make_pair(partitionStreamId, event.GetCommittedOffset() - 1); + auto cookieIt = MaxOffsetToCookie.lower_bound(offsetPair); + std::vector<ui64> cookies; + auto end = cookieIt; + auto begin = end; + if (cookieIt != MaxOffsetToCookie.end() && cookieIt->first == offsetPair) { + cookies.push_back(cookieIt->second); + ++end; + } + while (cookieIt != MaxOffsetToCookie.begin()) { + --cookieIt; + if (cookieIt->first.first == partitionStreamId) { + cookies.push_back(cookieIt->second); + begin = cookieIt; + } else { + break; + } + } + if (begin != end) { + MaxOffsetToCookie.erase(begin, end); + } + + NPersQueue::TReadResponse resp; + auto& respCommit = *resp.MutableCommit(); + auto& partitionStreamCookies = PartitionStreamToCookies[partitionStreamId]; + auto& committedCookies = CookiesRequestedToCommit[partitionStreamId]; + for (auto committedCookieIt = cookies.rbegin(), committedCookieEnd = cookies.rend(); committedCookieIt != committedCookieEnd; ++committedCookieIt) { + const ui64 cookie = *committedCookieIt; + respCommit.AddCookie(cookie); + partitionStreamCookies.erase(cookie); + committedCookies.erase(cookie); + } + AddResponse(std::move(resp)); +} + +void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent&& event) { + if (Settings.UseLockSession) { + Y_VERIFY(PartitionStreamToUncommittedOffsets[event.GetPartitionStream()->GetPartitionStreamId()].Empty()); + + NPersQueue::TReadResponse resp; + auto& lockResp = *resp.MutableLock(); + const auto& partitionStream = event.GetPartitionStream(); + lockResp.SetTopic(MakeLegacyTopicName(partitionStream)); + lockResp.SetPartition(partitionStream->GetPartitionId()); + lockResp.SetReadOffset(event.GetCommittedOffset()); + lockResp.SetEndOffset(event.GetEndOffset()); + lockResp.SetGeneration(partitionStream->GetPartitionStreamId()); + NThreading::TPromise<TLockInfo> confirmPromise = NThreading::NewPromise<TLockInfo>(); + confirmPromise.GetFuture().Subscribe([event = std::move(event)](const NThreading::TFuture<TLockInfo>& infoFuture) mutable { + const TLockInfo& info = infoFuture.GetValue(); + event.Confirm(info.ReadOffset, info.CommitOffset); + }); // It doesn't matter in what thread this callback will be called. + AddResponse(std::move(resp), std::move(confirmPromise)); + } else { + event.Confirm(); + } +} + +void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent&& event) { + const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); + CurrentPartitionStreams[partitionStreamId] = event.GetPartitionStream(); + if (Settings.UseLockSession) { + const auto partitionStream = event.GetPartitionStream(); + + Y_VERIFY(UnconfirmedDestroys.find(partitionStreamId) == UnconfirmedDestroys.end()); + if (PartitionStreamToUncommittedOffsets[partitionStreamId].Empty() || Settings.BalanceRightNow) { + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + event.Confirm(); + } else { + UnconfirmedDestroys.emplace(partitionStreamId, std::move(event)); + } + + NPersQueue::TReadResponse resp; + auto& releaseResp = *resp.MutableRelease(); + releaseResp.SetTopic(MakeLegacyTopicName(partitionStream)); + releaseResp.SetPartition(partitionStream->GetPartitionId()); + releaseResp.SetCanCommit(true); + releaseResp.SetGeneration(partitionStream->GetPartitionStreamId()); + AddResponse(std::move(resp)); + } else { + event.Confirm(); + } +} + +void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&& event) { + NPersQueue::TReadResponse resp; + auto& statusResp = *resp.MutablePartitionStatus(); + const auto& partitionStream = event.GetPartitionStream(); + statusResp.SetGeneration(partitionStream->GetPartitionStreamId()); + statusResp.SetTopic(MakeLegacyTopicName(partitionStream)); + statusResp.SetPartition(partitionStream->GetPartitionId()); + statusResp.SetCommittedOffset(event.GetCommittedOffset()); + statusResp.SetEndOffset(event.GetEndOffset()); + statusResp.SetWriteWatermarkMs(event.GetWriteWatermark().MilliSeconds()); + AddResponse(std::move(resp)); +} + +void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&& event) { + const ui64 partitionStreamId = event.GetPartitionStream()->GetPartitionStreamId(); + CurrentPartitionStreams.erase(partitionStreamId); + + { + NPersQueue::TReadResponse resp; + auto& respCommit = *resp.MutableCommit(); + for (ui64 cookie : CookiesRequestedToCommit[partitionStreamId]) { + respCommit.AddCookie(cookie); + } + if (resp.GetCommit().CookieSize()) { + AddResponse(std::move(resp)); + } + CookiesRequestedToCommit.erase(partitionStreamId); + } + + { + for (ui64 cookie : PartitionStreamToCookies[partitionStreamId]) { + CookieToOffsets.erase(cookie); + } + PartitionStreamToCookies.erase(partitionStreamId); + } + + { + auto begin = MaxOffsetToCookie.lower_bound(std::make_pair(partitionStreamId, 0)); // The first available offset. + if (begin != MaxOffsetToCookie.end() && begin->first.first == partitionStreamId) { + auto end = begin; + while (end != MaxOffsetToCookie.end() && end->first.first == partitionStreamId) { + ++end; + } + MaxOffsetToCookie.erase(begin, end); + } + } + + if (Settings.UseLockSession) { + PartitionStreamToUncommittedOffsets.erase(partitionStreamId); + UnconfirmedDestroys.erase(partitionStreamId); + + if (event.GetReason() != NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent::EReason::DestroyConfirmedByUser) { + NPersQueue::TReadResponse resp; + auto& releaseResp = *resp.MutableRelease(); + const auto& partitionStream = event.GetPartitionStream(); + releaseResp.SetTopic(MakeLegacyTopicName(partitionStream)); + releaseResp.SetPartition(partitionStream->GetPartitionId()); + releaseResp.SetCanCommit(false); + releaseResp.SetGeneration(partitionStream->GetPartitionStreamId()); + AddResponse(std::move(resp)); + } + } +} + +void TYdbSdkCompatibilityConsumer::HandleEvent(NYdb::NPersQueue::TSessionClosedEvent&& event) { + Destroy(event.GetIssues().ToString(), ToErrorCode(event.GetStatus())); +} + +void TYdbSdkCompatibilityConsumer::AddResponse(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead) { + Responses.emplace(std::move(response), std::move(readyToRead)); +} + +void TYdbSdkCompatibilityConsumer::AddResponse(TReadResponse&& response) { + Responses.emplace(std::move(response)); +} + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h new file mode 100644 index 0000000000..0ff6756686 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/impl/ydb_sdk_consumer.h @@ -0,0 +1,75 @@ +#pragma once +#include "iconsumer_p.h" + +#include <kikimr/public/sdk/cpp/client/ydb_persqueue/persqueue.h> + +#include <library/cpp/containers/disjoint_interval_tree/disjoint_interval_tree.h> + +#include <queue> + +namespace NPersQueue { + +class TYdbSdkCompatibilityConsumer : public IConsumerImpl, + public std::enable_shared_from_this<TYdbSdkCompatibilityConsumer> +{ +public: + TYdbSdkCompatibilityConsumer(const TConsumerSettings& settings, + std::shared_ptr<void> destroyEventRef, + TIntrusivePtr<TPQLibPrivate> pqLib, + TIntrusivePtr<ILogger> logger, + NYdb::NPersQueue::TPersQueueClient& client); + ~TYdbSdkCompatibilityConsumer(); + + void Init() override; + + NThreading::TFuture<TConsumerCreateResponse> Start(TInstant) noexcept override; + NThreading::TFuture<TError> IsDead() noexcept override; + void GetNextMessage(NThreading::TPromise<TConsumerMessage>& promise) noexcept override; + void Commit(const TVector<ui64>& cookies) noexcept override; + void Cancel() override; + void RequestPartitionStatus(const TString& topic, ui64 partition, ui64 generation) noexcept override; + +private: + class TReadSessionEventVisitor; + + NYdb::NPersQueue::TReadSessionSettings MakeReadSessionSettings(); + void SubscribeToNextEvent(); + void OnReadSessionEvent(); + void AnswerToRequests(); + void Destroy(const TError& description); + void Destroy(const TString& description, NErrorCode::EErrorCode code = NErrorCode::ERROR); + + void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent&& event); + void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCommitAcknowledgementEvent&& event); + void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TCreatePartitionStreamEvent&& event); + void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent&& event); + void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamStatusEvent&& event); + void HandleEvent(NYdb::NPersQueue::TReadSessionEvent::TPartitionStreamClosedEvent&& event); + void HandleEvent(NYdb::NPersQueue::TSessionClosedEvent&& event); + + void AddResponse(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead); + void AddResponse(TReadResponse&& response); + +private: + const TConsumerSettings Settings; + NYdb::NPersQueue::TReadSessionSettings YdbSdkSettings; + TIntrusivePtr<ILogger> Logger; + TString SessionId; + NYdb::NPersQueue::TPersQueueClient& Client; + std::shared_ptr<NYdb::NPersQueue::IReadSession> ReadSession; + NThreading::TPromise<TError> DeadPromise = NThreading::NewPromise<TError>(); + std::queue<NThreading::TPromise<TConsumerMessage>> Requests; + std::queue<TConsumerMessage> Responses; + bool SubscribedToNextEvent = false; + THashMap<ui64, std::pair<NYdb::NPersQueue::TDeferredCommit, ui64>> CookieToOffsets; // Cookie -> { commit, partition stream id }. + TMap<std::pair<ui64, ui64>, ui64> MaxOffsetToCookie; // { partition stream id, max offset of cookie } -> cookie. + THashMap<ui64, THashSet<ui64>> CookiesRequestedToCommit; // Partition stream id -> cookies that user requested to commit. + THashMap<ui64, THashSet<ui64>> PartitionStreamToCookies; // Partition stream id -> cookies. + ui64 NextCookie = 1; + // Commits for graceful release partition after commit. + THashMap<ui64, TDisjointIntervalTree<ui64>> PartitionStreamToUncommittedOffsets; // Partition stream id -> set of offsets. + THashMap<ui64, NYdb::NPersQueue::TReadSessionEvent::TDestroyPartitionStreamEvent> UnconfirmedDestroys; // Partition stream id -> destroy events. + THashMap<ui64, NYdb::NPersQueue::TPartitionStream::TPtr> CurrentPartitionStreams; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h new file mode 100644 index 0000000000..8b429053fd --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/iprocessor.h @@ -0,0 +1,61 @@ +#pragma once + +#include "types.h" +#include "responses.h" + +#include <library/cpp/threading/future/future.h> + +#include <util/generic/map.h> +#include <util/generic/vector.h> + +namespace NPersQueue { + +struct TPartition { + TString Topic; + ui32 PartitionId = 0; + + bool operator <(const TPartition& other) const { + return std::tie(Topic, PartitionId) < std::tie(other.Topic, other.PartitionId); + } + + explicit TPartition(const TString& topic, const ui32 partitionId) + : Topic(topic) + , PartitionId(partitionId) + {} +}; + +struct TProcessedMessage { + TString Topic; + TString Data; + TString SourceIdPrefix; + ui32 Group = 0; +}; + +struct TProcessedData { + TVector<TProcessedMessage> Messages; // maybe grouped by topic? +}; + +struct TOriginMessage { + TReadResponse::TData::TMessage Message; + NThreading::TPromise<TProcessedData> Processed; +}; + +struct TOriginData { + TMap<TPartition, TVector<TOriginMessage>> Messages; +}; + + +class IProcessor { +public: + virtual ~IProcessor() = default; + + // Returns data and promise for each message. + // Client MUST process each message individually in a deterministic way. + // Each original message CAN be transformed into several messages and written to several topics/sourceids. + // But for any given original message all pairs (topic, sourceIdPrefix) of resulting messages MUST be unique. + // Client MUST finish processing by signaling corresponding promise. + // Otherwise, exactly-once guarantees cannot be hold. + virtual NThreading::TFuture<TOriginData> GetNextData() noexcept = 0; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h new file mode 100644 index 0000000000..cbb221e9ee --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/iproducer.h @@ -0,0 +1,35 @@ +#pragma once + +#include "types.h" +#include "responses.h" +#include <library/cpp/threading/future/future.h> + +namespace NPersQueue { + +// IProducer is threadsafe. +// If one creates several producers with same SourceId in same topic - first Producer (from point of server view) will die. +// There could be only one producer with concrete SourceId in concrete topic at once. + +class IProducer { +public: + // Start producer. + // Producer can be used after its start will be finished. + virtual NThreading::TFuture<TProducerCreateResponse> Start(TInstant deadline = TInstant::Max()) noexcept = 0; + + NThreading::TFuture<TProducerCreateResponse> Start(TDuration timeout) noexcept { + return Start(TInstant::Now() + timeout); + } + + // Add write request to queue. + virtual NThreading::TFuture<TProducerCommitResponse> Write(TProducerSeqNo seqNo, TData data) noexcept = 0; + + // Add write request to queue without specifying seqNo. So, without deduplication (at least once guarantee). + virtual NThreading::TFuture<TProducerCommitResponse> Write(TData data) noexcept = 0; + + // Get future that is signalled when producer is dead. + virtual NThreading::TFuture<TError> IsDead() noexcept = 0; + + virtual ~IProducer() = default; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h new file mode 100644 index 0000000000..bb3320d795 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/logger.h @@ -0,0 +1,42 @@ +#pragma once +#include <util/generic/ptr.h> +#include <util/generic/string.h> + +namespace NPersQueue { + +class ILogger : public TAtomicRefCount<ILogger> { +public: + virtual ~ILogger() = default; + //level = syslog level + virtual void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) = 0; + virtual bool IsEnabled(int level) const = 0; +}; + +class TCerrLogger : public ILogger { +public: + explicit TCerrLogger(int level) + : Level(level) + { + } + + ~TCerrLogger() override = default; + + void Log(const TString& msg, const TString& sourceId, const TString& sessionId, int level) override; + + bool IsEnabled(int level) const override + { + return level <= Level; + } + + int GetLevel() const { + return Level; + } + +private: + static TStringBuf LevelToString(int level); + +private: + int Level; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h new file mode 100644 index 0000000000..1b1f7646c8 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h @@ -0,0 +1,50 @@ +#pragma once + +#include "types.h" +#include "iproducer.h" +#include "iconsumer.h" +#include "iprocessor.h" +#include "responses.h" + +#include <library/cpp/threading/future/future.h> + +#include <util/generic/noncopyable.h> +#include <util/generic/ptr.h> + +namespace NPersQueue { + +class TPQLibPrivate; + +class TPQLib : public TNonCopyable { +public: + explicit TPQLib(const TPQLibSettings& settings = TPQLibSettings()); + ~TPQLib(); + + // Deprecated flag means that PQLib doen't wait its objects during destruction. + // This behaviour leads to potential thread leakage and thread sanitizer errors. + // It is recommended to specify deprecated = false. In that case PQLib will cancel + // all objects and correctly wait for its threads in destructor. But, unfortunately, + // we can't migrate all current clients to this default behaviour automatically, + // because this can break existing programs unpredictably. + + // Producers creation + THolder<IProducer> CreateProducer(const TProducerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); + THolder<IProducer> CreateMultiClusterProducer(const TMultiClusterProducerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); + + // Consumers creation + THolder<IConsumer> CreateConsumer(const TConsumerSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); + + // Processors creation + THolder<IProcessor> CreateProcessor(const TProcessorSettings& settings, TIntrusivePtr<ILogger> logger = nullptr, bool deprecated = false); + + void SetLogger(TIntrusivePtr<ILogger> logger); + + TString GetUserAgent() const; + void SetUserAgent(const TString& userAgent); + +private: + TIntrusivePtr<TPQLibPrivate> Impl; + TAtomic Alive = 1; // Debug check for valid PQLib usage. +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h new file mode 100644 index 0000000000..de5a801172 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/responses.h @@ -0,0 +1,106 @@ +#pragma once + +#include "types.h" + +#include <library/cpp/threading/future/future.h> +#include <util/generic/string.h> + +namespace NPersQueue { + +struct TProducerCreateResponse { + TProducerCreateResponse(TWriteResponse&& response) + : Response(std::move(response)) + { + } + + TWriteResponse Response; +}; + +struct TProducerCommitResponse { + TProducerCommitResponse(TProducerSeqNo seqNo, TData data, TWriteResponse&& response) + : Response(std::move(response)) + , Data(std::move(data)) + , SeqNo(seqNo) + { + } + + TWriteResponse Response; + TData Data; + TProducerSeqNo SeqNo; +}; + + +struct TConsumerCreateResponse { + TConsumerCreateResponse(TReadResponse&& response) + : Response(std::move(response)) + { + } + + //will contain Error or Init + TReadResponse Response; +}; + +enum EMessageType { + EMT_LOCK, + EMT_RELEASE, + EMT_DATA, + EMT_ERROR, + EMT_STATUS, + EMT_COMMIT +}; + +struct TLockInfo { + ui64 ReadOffset = 0; + ui64 CommitOffset = 0; + bool VerifyReadOffset = false; + + TLockInfo() = default; + + // compatibility with msvc2015 + TLockInfo(ui64 readOffset, ui64 commitOffset, bool verifyReadOffset) + : ReadOffset(readOffset) + , CommitOffset(commitOffset) + , VerifyReadOffset(verifyReadOffset) + {} +}; + +static EMessageType GetType(const TReadResponse& response) { + if (response.HasData()) return EMT_DATA; + if (response.HasError()) return EMT_ERROR; + if (response.HasCommit()) return EMT_COMMIT; + if (response.HasPartitionStatus()) return EMT_STATUS; + if (response.HasRelease()) return EMT_RELEASE; + if (response.HasLock()) return EMT_LOCK; + + // for no warn return anything. + return EMT_LOCK; +} + + + +struct TConsumerMessage { + EMessageType Type; + //for DATA/ERROR/LOCK/RELEASE/COMMIT/STATUS: + //will contain Error, Data, Lock or Release and be consistent with Type + TReadResponse Response; + + //for LOCK only: + mutable NThreading::TPromise<TLockInfo> ReadyToRead; + + TConsumerMessage(TReadResponse&& response, NThreading::TPromise<TLockInfo>&& readyToRead) + : Type(EMT_LOCK) + , Response(std::move(response)) + , ReadyToRead(std::move(readyToRead)) + { + Y_VERIFY(GetType(Response) == EMT_LOCK); + } + + + explicit TConsumerMessage(TReadResponse&& response) + : Type(GetType(response)) + , Response(std::move(response)) + { + } +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp new file mode 100644 index 0000000000..b30115e294 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/consumer/main.cpp @@ -0,0 +1,236 @@ +#include <util/datetime/base.h> +#include <util/stream/file.h> +#include <util/string/join.h> +#include <util/string/vector.h> + +#include <google/protobuf/message.h> + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +using namespace NPersQueue; + + +int main() { + + // [BEGIN create pqlib] + // Step 1. First create logger. It could be your logger inherited from ILogger interface or one from lib. + TIntrusivePtr<ILogger> logger = new TCerrLogger(7); + + // Step 2. Create settings of main TPQLib object. + TPQLibSettings pqLibSettings; + + pqLibSettings.DefaultLogger = logger; + + // Step 3. Create TPQLib object. + TPQLib pq(pqLibSettings); + // [END create pqlib] + + // [BEGIN create consumer] + // Step 4. Parameters used in sample. + + // List of topics to read. + TVector<TString> topics = {"topic_path"}; + + // Path of consumer that will read topics. + TString consumerPath = "consumer_path"; + + // OAuth token. + TString oauthToken = "oauth_token"; + + // IAM JWT key + TString jwtKey = "<jwt json key>"; + + // TVM parameters. + TString tvmSecret = "tvm_secret"; + ui32 tvmClientId = 200000; + ui32 tvmDstId = 2001147; // logbroker main prestable + tvmDstId = 2001059; // logbroker main production + + // Logbroker endpoint. + TString endpoint = "man.logbroker.yandex.net"; + + // Simple single read. + + // Step 5. Create settings of consumer object. + TConsumerSettings settings; + // Fill consumer settings. See types.h for more information. + // Settings without defaults, must be set always. + settings.Server.Address = endpoint; + { + // Working with logbroker in Yandex.Cloud: + // Fill database for Yandex.Cloud Logbroker. + settings.Server.Database = "/Root"; + // Enable TLS. + settings.Server.EnableSecureConnection(""); + } + settings.Topics = topics; + settings.ClientId = consumerPath; + + // One must set CredentialsProvider for authentification. There could be OAuth and TVM providers. + // Create OAuth provider from OAuth ticket. + settings.CredentialsProvider = CreateOAuthCredentialsProvider(oauthToken); + // Or create TVM provider from TVM settings. + settings.CredentialsProvider = CreateTVMCredentialsProvider(tvmSecret, tvmClientId, tvmDstId, logger); + // Or create IAM provider from jwt-key + settings.CredentialsProvider = CreateIAMJwtParamsCredentialsForwarder(jwtKey, logger); + + // Step 6. Create consumer object. PQLib object must be valid during all lifetime of consumer, otherwise reading from consumer will fail but this is not UB. + auto consumer = pq.CreateConsumer(settings, logger); + + // Step 7. Start consuming data. + auto future = consumer->Start(); + + // Step 9. Wait initialization and check it result. + future.Wait(); + + if (future.GetValue().Response.HasError()) { + return 1; + } + // [END create consumer] + + // [BEGIN read messages] + // Step 10. Simple single read. + // Step 10.1. Request message from consumer. + auto msg = consumer->GetNextMessage(); + + // Step 10.2. Wait for message. + msg.Wait(); + + auto value = msg.GetValue(); + + // Step 10.3. Check result. + switch (value.Type) { + // Step 10.3.1. Result is error - consumer is in incorrect state now. You can log error and recreate everything starting from step 5. + case EMT_ERROR: + return 1; + // Step 10.3.2. Read data. + case EMT_DATA: { + // Process received messages. Inside will be several batches. + for (const auto& t : value.Response.data().message_batch()) { + const TString topic = t.topic(); + const ui32 partition = t.partition(); + Y_UNUSED(topic); + Y_UNUSED(partition); + // There are several messages from partition of topic inside one batch. + for (const auto& m : t.message()) { + const ui64 offset = m.offset(); + const TString& data = m.data(); + // Each message has offset in partition, payload data (decompressed or as is) and metadata. + Y_UNUSED(offset); + Y_UNUSED(data); + } + } + + // Step 10.3.3. Each processed data batch from step 10.3.2 must be committed once all data inside is processed. + + // This must be done using cookie from data batch. + auto cookie = msg.GetValue().Response.GetData().GetCookie(); + // Commit cookie['s]. Acknowledge from server will be received inside one of next messages from consumer later. + consumer->Commit({cookie}); + break; + } + // Step 10.3.4. Commit acknowledge. Can be used for logging and debugging purposes. (Can't be first message, only possible if one makes several + // consecutive reads. + case EMT_COMMIT: + break; + // Handle other cases - only for compiling, there could not be any. + default: + break; + + } + // [END read messages] + + // [BEGIN advanced read messages] + + // Point that consumer will perform advanced reads. + settings.UseLockSession = true; + + { + // Step 11. Create consumer object. PQLib object must be valid during all lifetime of consumer, otherwise reading from consumer will fail but this is not UB. + auto consumer = pq.CreateConsumer(settings, logger); + + // Step 12. Start consuming data. + auto future = consumer->Start(); + + // Step 14. Wait initialization and check it result. + future.Wait(); + + if (future.GetValue().Response.HasError()) { + return 1; + } + + // Step 14. Event loop. + while (true) { + // Step 14.1. Request message from consumer. + auto msg = consumer->GetNextMessage(); + + // Step 14.2. Wait for message. + msg.Wait(); + + const auto& value = msg.GetValue(); + + // Step 14.3. Check result. + switch (value.Type) { + // Step 14.3.1. Result is error: consumer is in incorrect state now. You can log error and recreate everything starting from step 11. + case EMT_ERROR: + return 1; + // Step 14.3.2. Lock request from server. + case EMT_LOCK: { + const TString& topic = value.Response.lock().topic(); + const ui32 partition = value.Response.lock().partition(); + // Server is ready to send data from corresponding partition of topic. Topic is legacy topic name = rt3.<cluster>--<converted topic path> + Y_UNUSED(topic); + Y_UNUSED(partition); + // One must react on this event with setting promise ReadyToRead right now or later. Only after this client will start receiving data from this partition. + msg.GetValue().ReadyToRead.SetValue(TLockInfo{0 /*readOffset*/, 0 /*commitOffset*/, false /*verifyReadOffset*/}); + break; + } + // Step 14.3.3. Release request from server. + case EMT_RELEASE: { + const TString& topic = value.Response.release().topic(); + const ui32 partition = value.Response.release().partition(); + Y_UNUSED(topic); + Y_UNUSED(partition); + // // Server will not send any data from partition of topic until next Lock request is confirmed by client. + break; + } + // Step 14.3.4. Read data. + case EMT_DATA: { + // Process received messages. Inside will be several batches. The same as in the simple read. + for (const auto& t : value.Response.data().message_batch()) { + const TString topic = t.topic(); + const ui32 partition = t.partition(); + Y_UNUSED(topic); + Y_UNUSED(partition); + // There are several messages from partition of topic inside one batch. + for (const auto& m : t.message()) { + const ui64 offset = m.offset(); + const TString& data = m.data(); + // Each message has offset in partition, payload data (decompressed or as is) and metadata. + Y_UNUSED(offset); + Y_UNUSED(data); + } + } + + // Step 14.3.5. Each processed data batch from step 14.3.4 should be committed after all data inside is processed. + + // This must be done using cookie from data batch. + auto cookie = msg.GetValue().Response.GetData().GetCookie(); + // Commit cookie['s]. Acknowledge from server will be received inside message from consumer later. + consumer->Commit({cookie}); + break; + } + // Step 14.3.6. Commit acknowledge. Can be used for logging and debugging purposes. + case EMT_STATUS: + case EMT_COMMIT: + break; + } // switch() + } // while() + } + + // [END advanced read messages] + + // Step 15. End of working. Nothing special needs to be done - desctuction of objects only. + + return 0; +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp new file mode 100644 index 0000000000..c91838892b --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/samples/producer/main.cpp @@ -0,0 +1,174 @@ +#include <queue> + +#include <util/datetime/base.h> +#include <util/stream/file.h> +#include <util/string/join.h> +#include <util/string/vector.h> + +#include <google/protobuf/message.h> + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +using namespace NPersQueue; + + +int main() { + + + // [BEGIN create pqlib] + // Step 1. First create logger. It could be your logger inherited from ILogger interface or one from lib. + TIntrusivePtr<ILogger> logger = new TCerrLogger(7); + + // Step 2. Create settings of main TPQLib object. + TPQLibSettings pqLibSettings; + + // Number of threads for processing gRPC events. + pqLibSettings.ThreadsCount = 1; + // Number of threads for compression/decomression actions. + pqLibSettings.CompressionPoolThreads = 3; + // Default logger. Used inside consumers and producers if no logger provided. + pqLibSettings.DefaultLogger = logger; + + // Step 3. Create TPQLib object. + TPQLib pq(pqLibSettings); + // [END create pqlib] + + // [BEGIN create producer] + // Step 4. Initialize producer parameters + + // Topics to read. + TString topic = "topic_path"; + // Partitions group. 0 means any. + ui32 group = 0; + + // Source identifier of messages group. + TString sourceId = "source_id"; + + // OAuth token. + TString oauthToken = "oauth_token"; + + // IAM JWT key + TString jwtKey = "<jwt json key>"; + + // TVM parameters. + TString tvmSecret = "tvm_secret"; + ui32 tvmClientId = 200000; + ui32 tvmDstId = 2001147; // logbroker main prestable + tvmDstId = 2001059; // logbroker main production + + // Logbroker endpoint. + TString endpoint = "logbroker.yandex.net"; + + // Step 5. Put producer parameters into settings. + TProducerSettings settings; + // Logbroker endpoint. + settings.Server.Address = endpoint; + { + // Working with logbroker in Yandex.Cloud: + // Fill database for Yandex.Cloud Logbroker. + settings.Server.Database = "/Root"; + // Enable TLS. + settings.Server.EnableSecureConnection(""); + } + // Topic path. + settings.Topic = topic; + // SourceId of messages group. + settings.SourceId = sourceId; + // Partition group to write to. + settings.PartitionGroup = group; + // ReconnectOnFailure will gurantee that producer will retry writes in case of failre. + settings.ReconnectOnFailure = true; //retries on server errors + // Codec describes how to compress data. + settings.Codec = NPersQueueCommon::ECodec::GZIP; + + // One must set CredentialsProvider for authentification. There could be OAuth and TVM providers. + // Create OAuth provider from OAuth ticket. + settings.CredentialsProvider = CreateOAuthCredentialsProvider(oauthToken); + // Or create TVM provider from TVM settings. + settings.CredentialsProvider = CreateTVMCredentialsProvider(tvmSecret, tvmClientId, tvmDstId, logger); + // Or create IAM provider from jwt-key + settings.CredentialsProvider = CreateIAMJwtParamsCredentialsForwarder(jwtKey, logger); + + // Step 6. Create producer object. PQLib object must be valid during all lifetime of producer, otherwise writing to producer will fail but this is not UB. + auto producer = pq.CreateProducer(settings, logger); + + // Step 7. Start producer. + auto future = producer->Start(); + + // Step 8. Wait initialization and check it result. + future.Wait(); + + if (future.GetValue().Response.HasError()) { + return 1; + } + // [END create producer] + + // [BEGIN write message] + + // Step 9. Single write sample. + // Step 9.1. Form message. + // Creation timestamp. + TInstant timestamp = TInstant::Now(); + // Message payload. + TString payload = "abacaba"; + + TData data{payload, timestamp}; + // Corresponding messages's sequence number. + ui64 seqNo = 1; + + // Step 9.2. Write message. Until result future is signalled - message is stored inside PQLib memory. Future will be signalled with message payload and result of writing. + auto result = producer->Write(seqNo, data); + + // Step 9.3 Check writing result. + // Check all results. + result.Wait(); + Y_VERIFY(!result.GetValue().Response.HasError()); + + // Only when future is destroyed no memory is used by message. + + // [END write message] + + // [BEGIN write messages with inflight] + + // Step 10. Writing loop with in-flight. + // Max in-flight border, parameter. + ui32 maxInflyCount = 10; + // Queue for messages in-flight. + std::queue<NThreading::TFuture<TProducerCommitResponse>> v; + + while(true) { + // Creation timestamp. + TInstant timestamp = TInstant::Now(); + // Message payload. + TString payload = "abacaba"; + TData data{payload, timestamp}; + + // SeqNo must increase for consecutive messages. + ++seqNo; + + // Step 10.1. Write message. + auto result = producer->Write(seqNo, data); + // Step 10.2. Put future inside queue. + v.push(result); + // Step 10.3. If in-flight size is too big. + if (v.size() > maxInflyCount) { + // Step 10.4. Wait for first message in queue processing result, check it. + v.front().Wait(); + Y_VERIFY(!v.front().GetValue().Response.HasError()); + // Step 10.5. Drop first processed message. Only after memory of this message will be freed. + v.pop(); + } + } + + // Step 10.6. After processing loop one must wait for all not yet processed messages. + + while (!v.empty()) { + v.front().Wait(); + Y_VERIFY(!v.front().GetValue().Response.HasError()); + v.pop(); + } + + // [END write messages with inflight] + + return 0; +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h new file mode 100644 index 0000000000..37c4423d6f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/types.h @@ -0,0 +1,402 @@ +#pragma once +#include "credentials_provider.h" +#include "logger.h" +#include <kikimr/yndx/api/protos/persqueue.pb.h> + + +#include <library/cpp/logger/priority.h> + +#include <util/generic/string.h> +#include <util/generic/buffer.h> +#include <util/generic/maybe.h> +#include <util/generic/vector.h> +#include <util/generic/hash.h> +#include <util/datetime/base.h> +#include <util/stream/output.h> +#include <util/string/builder.h> + +#include <grpc++/channel.h> + +#include <limits> + +namespace NPersQueue { +using ECodec = NPersQueueCommon::ECodec; +using TCredentials = NPersQueueCommon::TCredentials; +using TError = NPersQueueCommon::TError; + +using TProducerSeqNo = ui64; // file offset or something with equivalent meaning. +using TConsumerOffset = i64; // partition offset. Have special value -1. All other values are >= 0. +using TCredProviderPtr = std::shared_ptr<ICredentialsProvider>; + +constexpr ui32 DEFAULT_MEMORY_USAGE = 100 << 20; //100mb +constexpr ui32 DEFAULT_READS_INFLY = 10; +constexpr ui32 DEFAULT_BATCH_SIZE = 1 << 20; //1mb +constexpr TDuration DEFAULT_CHANNEL_CREATION_TIMEOUT = TDuration::Seconds(10); +constexpr TDuration DEFAULT_RECONNECTION_DELAY = TDuration::MilliSeconds(10); +constexpr TDuration DEFAULT_MAX_RECONNECTION_DELAY = TDuration::Seconds(1); +constexpr TDuration DEFAULT_START_SESSION_TIMEOUT = TDuration::Seconds(30); +constexpr TDuration DEFAULT_CHECK_TOKEN_PERIOD = TDuration::Hours(1); +inline const THashMap<ECodec, TString> CodecIdByType{{ECodec::DEFAULT, TString(1, '\0')}, {ECodec::RAW, TString(1, '\0')}, {ECodec::GZIP, TString(1, '\1')}, {ECodec::LZOP, TString(1, '\2')}, {ECodec::ZSTD, TString(1, '\3')}}; + +struct TPQLibSettings { + size_t ThreadsCount = 1; + size_t CompressionPoolThreads = 2; + size_t GRpcThreads = 1; + TDuration ChannelCreationTimeout = DEFAULT_CHANNEL_CREATION_TIMEOUT; + TIntrusivePtr<ILogger> DefaultLogger = {}; + + bool EnableGRpcV1 = std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED"); +}; + +enum class EClusterDiscoveryUsageMode { + Auto, + Use, + DontUse, +}; + +// destination server +struct TServerSetting { + // Endpoint of logbroker cluster to connect. + // Possible variants: + // + // 1. lbkx cluster. + // Set address to lbkx.logbroker.yandex.net + // + // 2. logbroker federation. + // If you want to write/read to/from concrete cluster + // (with ReadMirroredPartitions or not, but without ReadFromAllClusterSources option), + // use the following addresses: + // iva.logbroker.yandex.net + // man.logbroker.yandex.net + // myt.logbroker.yandex.net + // sas.logbroker.yandex.net + // vla.logbroker.yandex.net + // + // If you create consumer with ReadFromAllClusterSources option, + // use logbroker.yandex.net + TString Address; + + ui16 Port = 2135; + TString Database = GetRootDatabase(); + EClusterDiscoveryUsageMode UseLogbrokerCDS = EClusterDiscoveryUsageMode::Auto; + bool UseSecureConnection = false; + TString CaCert = TString(); + + TServerSetting(TString address, ui16 port = 2135) + : Address(std::move(address)) + , Port(port) + { + Y_VERIFY(!Address.empty()); + Y_VERIFY(Port > 0); + } + + TServerSetting() = default; + TServerSetting(const TServerSetting&) = default; + TServerSetting(TServerSetting&&) = default; + + TServerSetting& operator=(const TServerSetting&) = default; + TServerSetting& operator=(TServerSetting&&) = default; + + TString GetFullAddressString() const { + return TStringBuilder() << Address << ':' << Port; + } + void EnableSecureConnection(const TString& caCert) { + UseSecureConnection = true; + CaCert = caCert; + } + void DisableSecureConnection() { + UseSecureConnection = false; + CaCert.clear(); + } + + const TString& GetRootDatabase(); + + bool operator < (const TServerSetting& rhs) const { + return std::tie(Address, Port, Database, UseSecureConnection) < std::tie(rhs.Address, rhs.Port, rhs.Database, rhs.UseSecureConnection); + } +}; + +// +// consumer/producer settings +// + +struct TProducerSettings { + TServerSetting Server; + std::shared_ptr<ICredentialsProvider> CredentialsProvider = CreateInsecureCredentialsProvider(); + + TString Topic; + TString SourceId; + + ui32 PartitionGroup = 0; + ECodec Codec = ECodec::GZIP; + int Quality = -1; + THashMap<TString, TString> ExtraAttrs; + + // Should we reconnect when session breaks? + bool ReconnectOnFailure = false; + + // + // Setting for reconnection + // + + // In case of lost connection producer will try + // to reconnect and resend data <= MaxAttempts times. + // Then producer will be dead. + unsigned MaxAttempts = std::numeric_limits<unsigned>::max(); + + // Time delay before next producer reconnection attempt. + // The second attempt will be in 2 * ReconnectionDelay, + // the third - in 4 * ReconnectionDelay, etc. Maximum value + // of delay is MaxReconnectionDelay. + TDuration ReconnectionDelay = DEFAULT_RECONNECTION_DELAY; + TDuration MaxReconnectionDelay = DEFAULT_MAX_RECONNECTION_DELAY; + + // Timeout for session initialization operation. + TDuration StartSessionTimeout = DEFAULT_START_SESSION_TIMEOUT; + // Producer will check token from 'ICredentialsProvider' with this period and if changed - send to server as soon as possible. + // Available only with gRPC data plane API v1. + TDuration CheckTokenPeriod = DEFAULT_CHECK_TOKEN_PERIOD; + + // Preferred cluster to write. + // This setting is used only in case of cluster discovery is used. + // Case insensitive. + // If PreferredCluster is not available, + // producer will connect to other cluster. + TString PreferredCluster; + + bool DisableCDS = false; // only for grpcv1 mode +}; + +struct TMultiClusterProducerSettings { + struct TServerWeight { + // Setting of producer connected to concrete DC. + // ReconnectOnFailure must be true. + // MaxAttempts must be a finite number. + // SourceId will be complemented with source id prefix and producer type. Can be empty. + // There is no guarantee that messages will arrive in the order of writes in case of one DC is absent. + TProducerSettings ProducerSettings; + unsigned Weight; + + explicit TServerWeight(TProducerSettings producerSettings, unsigned weight = 1) + : ProducerSettings(std::move(producerSettings)) + , Weight(weight) + { + Y_VERIFY(Weight > 0); + } + + TServerWeight() = default; + TServerWeight(const TServerWeight&) = default; + TServerWeight(TServerWeight&&) = default; + + TServerWeight& operator=(const TServerWeight&) = default; + TServerWeight& operator=(TServerWeight&&) = default; + }; + + std::vector<TServerWeight> ServerWeights; + size_t MinimumWorkingDcsCount = 1; // Minimum Dcs to start and not to die. + TString SourceIdPrefix; // There is no guarantee that messages will arrive in the order of writes in case of one DC is absent. +}; + +struct TConsumerSettings { + TVector<TString> Topics; + bool ReadMirroredPartitions = true; + + // Discovers all clusters where topic is presented and create consumer that reads from all of them. + // Conflicts with ReadMirroredPartitions option (should be set to "no"). + // Will ignore the following options: + // ReconnectOnFailure (forced to "yes") + // MaxAttempts (forced to infinity == std::numeric_limits<unsigned>::max()) + // Server.UseLogbrokerCDS (forced to "Use") + // + // Also see description of option ReconnectOnFailure. + bool ReadFromAllClusterSources = false; + + TServerSetting Server; + std::shared_ptr<ICredentialsProvider> CredentialsProvider = CreateInsecureCredentialsProvider(); + TString ClientId; + + bool UseLockSession = false; + ui32 PartsAtOnce = 0; + + bool Unpack = true; + bool SkipBrokenChunks = false; + + ui64 MaxMemoryUsage = DEFAULT_MEMORY_USAGE; + ui32 MaxInflyRequests = DEFAULT_READS_INFLY; + + // read settings + ui32 MaxCount = 0; // zero means unlimited + ui32 MaxSize = DEFAULT_BATCH_SIZE; // zero means unlimited + ui32 MaxTimeLagMs = 0; // zero means unlimited + ui64 ReadTimestampMs = 0; // read data from this timestamp + + ui32 MaxUncommittedCount = 0; // zero means unlimited + ui64 MaxUncommittedSize = 0; // zero means unlimited + + TVector<ui32> PartitionGroups; // groups to read + + bool PreferLocalProxy = false; + bool BalanceRightNow = false; + bool CommitsDisabled = false; + + // Should we reconnect when session breaks? + // + // Keep in mind that it is possible to receive + // message duplicates when reconnection occures. + bool ReconnectOnFailure = false; + + // + // Settings for reconnection + // + + // In case of lost connection consumer will try + // to reconnect and resend requests <= MaxAttempts times. + // Then consumer will be dead. + unsigned MaxAttempts = std::numeric_limits<unsigned>::max(); + + // Time delay before next consumer reconnection attempt. + // The second attempt will be in 2 * ReconnectionDelay, + // the third - in 3 * ReconnectionDelay, etc. Maximum value + // of delay is MaxReconnectionDelay. + TDuration ReconnectionDelay = DEFAULT_RECONNECTION_DELAY; + TDuration MaxReconnectionDelay = DEFAULT_MAX_RECONNECTION_DELAY; + TDuration StartSessionTimeout = DEFAULT_START_SESSION_TIMEOUT; + + bool DisableCDS = false; // only for grpcv1 mode + + bool UseV2RetryPolicyInCompatMode = false; +}; + +struct TProcessorSettings { + ui64 MaxOriginDataMemoryUsage; + ui64 MaxProcessedDataMemoryUsage; + TDuration SourceIdIdleTimeout; + + std::shared_ptr<ICredentialsProvider> CredentialsProvider; + + //UseLockSession parameter will be ignored + TConsumerSettings ConsumerSettings; + + //Topic and SourceId pararameters will be ignored + TProducerSettings ProducerSettings; + + TProcessorSettings() + : MaxOriginDataMemoryUsage(DEFAULT_MEMORY_USAGE) + , MaxProcessedDataMemoryUsage(DEFAULT_MEMORY_USAGE) + , SourceIdIdleTimeout(TDuration::Hours(1)) + , CredentialsProvider(CreateInsecureCredentialsProvider()) + { + ConsumerSettings.CredentialsProvider = ProducerSettings.CredentialsProvider = CredentialsProvider; + } +}; + +class TData { +public: + // empty: not for passing to producer + TData() = default; + + // implicit constructor from source data to be encoded later by producer with default codec from settings + // or explicitly specified one + TData(TString sourceData, ECodec codec = ECodec::DEFAULT, TInstant timestamp = TInstant::Now()) + : TData(timestamp, sourceData, codec == ECodec::RAW ? sourceData : TString(), codec, sourceData.size()) + { + Y_VERIFY(!SourceData.empty()); + Y_VERIFY(timestamp != TInstant::Zero()); + } + + TData(TString sourceData, TInstant timestamp) + : TData(std::move(sourceData), ECodec::DEFAULT, timestamp) + { + } + + // construct already encoded TData + static TData Encoded(TString encodedData, ECodec codec, TInstant timestamp = TInstant::Now(), ui32 originalSize = 0) { + Y_VERIFY(!encodedData.empty()); + Y_VERIFY(codec != ECodec::RAW && codec != ECodec::DEFAULT); + Y_VERIFY(timestamp != TInstant::Zero()); + return TData(timestamp, TString(encodedData), encodedData, codec, originalSize == 0 ? encodedData.size() * 10 : originalSize); + } + + // construct user defined raw data that shouldn't be encoded by producer + static TData Raw(TString rawData, TInstant timestamp = TInstant::Now()) { + Y_VERIFY(!rawData.empty()); + Y_VERIFY(timestamp != TInstant::Zero()); + return TData(timestamp, rawData, rawData, ECodec::RAW, rawData.size()); + } + + TData(const TData&) = default; + TData(TData&&) = default; + + TData& operator=(const TData&) = default; + TData& operator=(TData&&) = default; + + ECodec GetCodecType() const { + return Codec; + } + + bool IsEncoded() const { + return !EncodedData.empty(); + } + + const TString& GetEncodedData() const { + Y_VERIFY(IsEncoded()); + return EncodedData; + } + + // gets some data for error report + const TString& GetSourceData() const { + return SourceData; + } + + TInstant GetTimestamp() const { + return Timestamp; + } + + bool Empty() const { + return GetTimestamp() == TInstant::Zero(); + } + + // encoding of data by default codec or data codec that is not encoded + // if data's codec is default, uses defaultCodec parameter, + // otherwise uses data's codec + static TData Encode(TData data, ECodec defaultCodec, int quality = -1); + + ui32 GetOriginalSize() const { + return OriginalSize; + } + + // special function to make raw enconding that doesn't need cpu time. + static TData MakeRawIfNotEncoded(TData data); + + bool operator==(const TData& data) const { + return Codec == data.Codec + && Timestamp == data.Timestamp + && SourceData == data.SourceData + && EncodedData == data.EncodedData; + } + + bool operator!=(const TData& data) const { + return !operator==(data); + } + +private: + TData(TInstant timestamp, TString sourceData, TString encodedData, ECodec codec, ui32 originalSize) + : Timestamp(timestamp) + , SourceData(std::move(sourceData)) + , EncodedData(std::move(encodedData)) + , Codec(codec) + , OriginalSize(originalSize) + { + } + + static THolder<IOutputStream> CreateCoder(ECodec codec, TData& data, int quality); + +private: + TInstant Timestamp; + TString SourceData; + TString EncodedData; + ECodec Codec = ECodec::DEFAULT; + ui32 OriginalSize = 0; +}; + +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt new file mode 100644 index 0000000000..ec1f7fb67f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/CMakeLists.txt @@ -0,0 +1,30 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(cpp-v2-ut_utils) +target_compile_options(cpp-v2-ut_utils PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(cpp-v2-ut_utils PUBLIC + contrib-libs-cxxsupp + yutil + deprecated-cpp-v2 + yndx-grpc_services-persqueue + yndx-persqueue-msgbus_server + cpp-grpc-server + cpp-testing-unittest + core-client-server + ydb-core-testlib + library-persqueue-topic_parser_public +) +target_sources(cpp-v2-ut_utils PRIVATE + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp + ${CMAKE_SOURCE_DIR}/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp +) diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp new file mode 100644 index 0000000000..6f201f39f5 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.cpp @@ -0,0 +1,263 @@ +#include "data_writer.h" + +namespace NPersQueue::NTests { +using namespace NKikimr; + +void TPQDataWriter::Read( + const TString& topic, const TString& clientId, const TString& ticket, bool error, bool checkACL, + bool useBatching, bool onlyCreate +) { + + grpc::ClientContext context; + + auto stream = StubP_->ReadSession(&context); + UNIT_ASSERT(stream); + + // Send initial request. + TReadRequest req; + TReadResponse resp; + + req.MutableInit()->AddTopics(topic); + + req.MutableInit()->SetClientId(clientId); + req.MutableInit()->SetProxyCookie(ProxyCookie); + if (!ticket.empty()) { + req.MutableCredentials()->SetTvmServiceTicket(ticket); + } + + if (useBatching) { + req.MutableInit()->SetProtocolVersion(TReadRequest::Batching); + } + + if (!stream->Write(req)) { + ythrow yexception() << "write fail"; + } + //TODO[komels]: why this leads to timeout? + //Server.AnnoyingClient->GetClientInfo({topic}, clientId, true); + UNIT_ASSERT(stream->Read(&resp)); + if (error) { + UNIT_ASSERT(resp.HasError()); + return; + } + UNIT_ASSERT_C(resp.HasInit(), resp); + + if (onlyCreate) + return; + + for (ui32 i = 0; i < 11; ++i) { + TReadRequest req; + + req.MutableRead()->SetMaxCount(1); + + if (!stream->Write(req)) { + ythrow yexception() << "write fail"; + } + Server.AnnoyingClient->AlterTopic(FullTopicName, i < 10 ? 2 : 3); + + } + + if (checkACL) { + NACLib::TDiffACL acl; + acl.RemoveAccess(NACLib::EAccessType::Allow, NACLib::SelectRow, clientId + "@" BUILTIN_ACL_DOMAIN); + Server.AnnoyingClient->ModifyACL("/Root/PQ", FullTopicName, acl.SerializeAsString()); + + TReadRequest req; + req.MutableRead()->SetMaxCount(1); + if (!stream->Write(req)) { + ythrow yexception() << "write fail"; + } + + UNIT_ASSERT(stream->Read(&resp)); + UNIT_ASSERT(resp.HasError() && resp.GetError().GetCode() == NPersQueue::NErrorCode::EErrorCode::ACCESS_DENIED); + return; + } + Server.AnnoyingClient->GetClientInfo({FullTopicName}, clientId, true); + for (ui32 i = 0; i < 11; ++i) { + TReadResponse resp; + + UNIT_ASSERT(stream->Read(&resp)); + + if (useBatching) { + UNIT_ASSERT(resp.HasBatchedData()); + UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().PartitionDataSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).BatchSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).MessageDataSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(resp.GetBatchedData().GetPartitionData(0).GetBatch(0).GetMessageData(0).GetOffset(), i); + } else { + UNIT_ASSERT(resp.HasData()); + UNIT_ASSERT_VALUES_EQUAL(resp.GetData().MessageBatchSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(resp.GetData().GetMessageBatch(0).MessageSize(), 1); + UNIT_ASSERT_VALUES_EQUAL(resp.GetData().GetMessageBatch(0).GetMessage(0).GetOffset(), i); + } + } + //TODO: check here that read will never done UNIT_ASSERT(!stream->Read(&resp)); + { + for (ui32 i = 1; i < 11; ++i) { + TReadRequest req; + + req.MutableCommit()->AddCookie(i); + + if (!stream->Write(req)) { + ythrow yexception() << "write fail"; + } + } + ui32 i = 1; + while (i <= 10) { + TReadResponse resp; + + UNIT_ASSERT(stream->Read(&resp)); + Cerr << resp << "\n"; + UNIT_ASSERT(resp.HasCommit()); + UNIT_ASSERT(resp.GetCommit().CookieSize() > 0); + for (ui32 j = 0; j < resp.GetCommit().CookieSize(); ++j) { + UNIT_ASSERT( resp.GetCommit().GetCookie(j) == i); + ++i; + UNIT_ASSERT(i <= 11); + } + } + Server.AnnoyingClient->GetClientInfo({FullTopicName}, clientId, true); + } +} + +void TPQDataWriter::WaitWritePQServiceInitialization() { + TWriteRequest req; + TWriteResponse resp; + while (true) { + grpc::ClientContext context; + + auto stream = StubP_->WriteSession(&context); + UNIT_ASSERT(stream); + req.MutableInit()->SetTopic(ShortTopicName); + req.MutableInit()->SetSourceId("12345678"); + req.MutableInit()->SetProxyCookie(ProxyCookie); + + if (!stream->Write(req)) { + continue; + } + UNIT_ASSERT(stream->Read(&resp)); + if (resp.GetError().GetCode() == NPersQueue::NErrorCode::INITIALIZING) { + Sleep(TDuration::MilliSeconds(50)); + } else { + break; + } + } +} + +ui32 TPQDataWriter::InitSession(const TString& sourceId, ui32 pg, bool success, ui32 step) { + TWriteRequest req; + TWriteResponse resp; + + grpc::ClientContext context; + + auto stream = StubP_->WriteSession(&context); + + UNIT_ASSERT(stream); + req.MutableInit()->SetTopic(ShortTopicName); + req.MutableInit()->SetSourceId(sourceId); + req.MutableInit()->SetPartitionGroup(pg); + req.MutableInit()->SetProxyCookie(ProxyCookie); + + UNIT_ASSERT(stream->Write(req)); + UNIT_ASSERT(stream->Read(&resp)); + Cerr << "Init result: " << resp << "\n"; + //TODO: ensure topic creation - proxy already knows about new partitions, but tablet - no! + if (!success) { + UNIT_ASSERT(resp.HasError()); + return 0; + } else { + if (!resp.HasInit() && step < 5) { + Sleep(TDuration::MilliSeconds(100)); + return InitSession(sourceId, pg, success, step + 1); + } + UNIT_ASSERT(resp.HasInit()); + return resp.GetInit().GetPartition(); + } + return 0; +} + +ui32 TPQDataWriter::WriteImpl( + const TString& topic, const TVector<TString>& data, bool error, const TString& ticket, bool batch +) { + grpc::ClientContext context; + + auto stream = StubP_->WriteSession(&context); + UNIT_ASSERT(stream); + + // Send initial request. + TWriteRequest req; + TWriteResponse resp; + + req.MutableInit()->SetTopic(topic); + req.MutableInit()->SetSourceId(SourceId_); + req.MutableInit()->SetProxyCookie(ProxyCookie); + if (!ticket.empty()) + req.MutableCredentials()->SetTvmServiceTicket(ticket); + auto item = req.MutableInit()->MutableExtraFields()->AddItems(); + item->SetKey("key"); + item->SetValue("value"); + + if (!stream->Write(req)) { + ythrow yexception() << "write fail"; + } + + ui32 part = 0; + + UNIT_ASSERT(stream->Read(&resp)); + + if (!error) { + UNIT_ASSERT_C(resp.HasInit(), resp); + UNIT_ASSERT_C(!resp.GetInit().GetSessionId().empty(), resp); + part = resp.GetInit().GetPartition(); + } else { + Cerr << resp << "\n"; + UNIT_ASSERT(resp.HasError()); + return 0; + } + + // Send data requests. + Flush(data, stream, ticket, batch); + + Flush(data, stream, ticket, batch); + + Flush(data, stream, ticket, batch); + + Flush(data, stream, ticket, batch); + + //will cause only 4 answers in stream->Read - third call will fail, not blocks + stream->WritesDone(); + + UNIT_ASSERT(!stream->Read(&resp)); + + auto status = stream->Finish(); + UNIT_ASSERT(status.ok()); + return part; +} + +ui64 TPQDataWriter::ReadCookieFromMetadata(const std::multimap<grpc::string_ref, grpc::string_ref>& meta) const { + auto ci = meta.find("cookie"); + if (ci == meta.end()) { + ythrow yexception() << "server didn't provide the cookie"; + } else { + return FromString<ui64>(TStringBuf(ci->second.data(), ci->second.size())); + } +} + +void TPQDataWriter::InitializeChannel() { + Channel_ = grpc::CreateChannel("[::1]:" + ToString(Server.GrpcPort), grpc::InsecureChannelCredentials()); + Stub_ = NKikimrClient::TGRpcServer::NewStub(Channel_); + + grpc::ClientContext context; + NKikimrClient::TChooseProxyRequest request; + NKikimrClient::TResponse response; + auto status = Stub_->ChooseProxy(&context, request, &response); + UNIT_ASSERT(status.ok()); + Cerr << response << "\n"; + UNIT_ASSERT(response.GetStatus() == NMsgBusProxy::MSTATUS_OK); + ProxyCookie = response.GetProxyCookie(); + Channel_ = grpc::CreateChannel( + "[" + response.GetProxyName() + "]:" + ToString(Server.GrpcPort), + grpc::InsecureChannelCredentials() + ); + StubP_ = NPersQueue::PersQueueService::NewStub(Channel_); +} +} // namespace NKikimr::NPersQueueTests diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h new file mode 100644 index 0000000000..c4e46ab08e --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/data_writer.h @@ -0,0 +1,117 @@ +#pragma once + +#include "test_server.h" + +#include <kikimr/yndx/api/grpc/persqueue.grpc.pb.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <grpc++/client_context.h> +#include <grpc++/create_channel.h> + +namespace NPersQueue::NTests { + +class TPQDataWriter { +public: + TPQDataWriter( + const TString& defaultTopicFullName, const TString& defaultShortTopicName, + const TString& sourceId, TTestServer& server + ) + : FullTopicName(defaultTopicFullName) + , ShortTopicName(defaultShortTopicName) + , SourceId_(sourceId) + , Server(server) + { + InitializeChannel(); + WaitWritePQServiceInitialization(); + } + + void Read(const TString& topic, const TString& clientId, const TString& ticket = "", bool error = false, + bool checkACL = false, bool useBatching = false, bool onlyCreate = false); + + void WaitWritePQServiceInitialization(); + + ui32 InitSession(const TString& sourceId, ui32 pg, bool success, ui32 step = 0); + + ui32 Write(const TString& topic, const TString& data, bool error = false, const TString& ticket = "") { + return WriteImpl(topic, {data}, error, ticket, false); + } + + ui32 WriteBatch(const TString& topic, const TVector<TString>& data, bool error = false, + const TString& ticket = "") { + return WriteImpl(topic, data, error, ticket, true); + } + +private: + ui32 WriteImpl(const TString& topic, const TVector<TString>& data, bool error, const TString& ticket, + bool batch); + + template<typename S> + void Flush(const TVector<TString>& data, S& stream, const TString& ticket, bool batch) { + TWriteRequest request; + TWriteResponse response; + + TVector<ui64> allSeqNo; + if (batch) { + for (const TString& d: data) { + ui64 seqNo = AtomicIncrement(SeqNo_); + allSeqNo.push_back(seqNo); + auto *mutableData = request.MutableDataBatch()->AddData(); + mutableData->SetSeqNo(seqNo); + mutableData->SetData(d); + } + } else { + ui64 seqNo = AtomicIncrement(SeqNo_); + allSeqNo.push_back(seqNo); + request.MutableData()->SetSeqNo(seqNo); + request.MutableData()->SetData(JoinSeq("\n", data)); + } + if (!ticket.empty()) { + request.MutableCredentials()->SetTvmServiceTicket(ticket); + } + + Cerr << "request: " << request << Endl; + if (!stream->Write(request)) { + ythrow yexception() << "write fail"; + } + + UNIT_ASSERT(stream->Read(&response)); + if (batch) { + UNIT_ASSERT_C(response.HasAckBatch(), response); + UNIT_ASSERT_VALUES_EQUAL(data.size(), response.GetAckBatch().AckSize()); + for (size_t i = 0; i < data.size(); ++i) { + const auto& ack = response.GetAckBatch().GetAck(i); + UNIT_ASSERT(!ack.GetAlreadyWritten()); + UNIT_ASSERT(!ack.HasStat()); + UNIT_ASSERT_VALUES_EQUAL(ack.GetSeqNo(), allSeqNo[i]); + } + UNIT_ASSERT(response.GetAckBatch().HasStat()); + } else { + const auto& ack = response.GetAck(); + UNIT_ASSERT(!ack.GetAlreadyWritten()); + UNIT_ASSERT_VALUES_EQUAL(ack.GetSeqNo(), allSeqNo[0]); + UNIT_ASSERT(ack.HasStat()); + } + } + + ui64 ReadCookieFromMetadata(const std::multimap<grpc::string_ref, grpc::string_ref>& meta) const; + + void InitializeChannel(); + +private: + TString FullTopicName; + TString ShortTopicName; + const TString SourceId_; + + TTestServer& Server; + + TAtomic SeqNo_ = 1; + + //! Сетевой канал взаимодействия с proxy-сервером. + std::shared_ptr<grpc::Channel> Channel_; + std::unique_ptr<NKikimrClient::TGRpcServer::Stub> Stub_; + std::unique_ptr<PersQueueService::Stub> StubP_; + + ui64 ProxyCookie = 0; +}; +} // NKikimr::NPersQueueTests diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h new file mode 100644 index 0000000000..9f0712933a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h @@ -0,0 +1,323 @@ +#pragma once +#include "test_server.h" +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/types.h> +#include <ydb/library/persqueue/topic_parser_public/topic_parser.h> +#include <library/cpp/logger/log.h> +#include <util/system/tempfile.h> + +#define TEST_CASE_NAME (this->Name_) + +namespace NPersQueue { +class SDKTestSetup { +protected: + TString TestCaseName; + + THolder<TTempFileHandle> NetDataFile; + THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> DataCenters; + TString LocalDC = "dc1"; + TTestServer Server = TTestServer(false /* don't start */); + + TLog Log = TLog("cerr"); + + TPQLibSettings PQLibSettings; + THolder<TPQLib> PQLib; + +public: + SDKTestSetup(const TString& testCaseName, bool start = true) + : TestCaseName(testCaseName) + { + InitOptions(); + if (start) { + Start(); + } + } + + void InitOptions() { + Log.SetFormatter([testCaseName = TestCaseName](ELogPriority priority, TStringBuf message) { + return TStringBuilder() << TInstant::Now() << " :" << testCaseName << " " << priority << ": " << message << Endl; + }); + Server.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::Max()); + // Default TTestServer value for 'MaxReadCookies' is 10. With this value the tests are flapping with two errors: + // 1. 'got more than 10 unordered cookies to commit 12' + // 2. 'got more than 10 uncommitted reads' + Server.ServerSettings.PQConfig.Clear(); + Server.ServerSettings.PQConfig.SetEnabled(true); + Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1); + Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1); + Server.ServerSettings.PQClusterDiscoveryConfig.SetEnabled(true); + SetNetDataViaFile("::1/128\t" + GetLocalCluster()); + + auto seed = TInstant::Now().MicroSeconds(); + // This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable + Log << TLOG_INFO << "Random seed for debugging is " << seed; + std::srand(seed); + } + + void Start(bool waitInit = true, bool addBrokenDatacenter = !std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED")) { + Server.StartServer(false); + //Server.EnableLogs({NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_READ_PROXY}); + Server.AnnoyingClient->InitRoot(); + if (DataCenters.empty()) { + THashMap<TString, NKikimr::NPersQueueTests::TPQTestClusterInfo> dataCenters; + dataCenters.emplace("dc1", NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true}); + if (addBrokenDatacenter) { + dataCenters.emplace("dc2", NKikimr::NPersQueueTests::TPQTestClusterInfo{"dc2.logbroker.yandex.net", false}); + } + Server.AnnoyingClient->InitDCs(dataCenters); + } else { + Server.AnnoyingClient->InitDCs(DataCenters, LocalDC); + } + Server.AnnoyingClient->InitSourceIds(); + CreateTopic(GetTestTopic(), GetLocalCluster()); + if (waitInit) { + Server.WaitInit(GetTestTopic()); + } + PQLibSettings.DefaultLogger = MakeIntrusive<TCerrLogger>(TLOG_DEBUG); + PQLib = MakeHolder<TPQLib>(PQLibSettings); + } + + THolder<TPQLib>& GetPQLib() { + return PQLib; + } + + const TPQLibSettings& GetPQLibSettings() const { + return PQLibSettings; + } + + TString GetTestTopic() const { + return "topic1"; + } + + TString GetTestClient() const { + return "test-reader"; + } + + TString GetTestMessageGroupId() const { + return "test-message-group-id"; + } + + TString GetLocalCluster() const { + return LocalDC; + } + + + NGrpc::TServerOptions& GetGrpcServerOptions() { + return Server.GrpcServerOptions; + } + + void SetNetDataViaFile(const TString& netDataTsv) { + NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); + NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size()); + NetDataFile->FlushData(); + Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); + } + + TProducerSettings GetProducerSettings() const { + TProducerSettings producerSettings; + producerSettings.Topic = GetTestTopic(); + producerSettings.SourceId = GetTestMessageGroupId(); + producerSettings.Server = TServerSetting{"localhost", Server.GrpcPort}; + producerSettings.Server.Database = "/Root"; + return producerSettings; + } + + TConsumerSettings GetConsumerSettings() const { + TConsumerSettings consumerSettings; + consumerSettings.Topics = {GetTestTopic()}; + consumerSettings.ClientId = GetTestClient(); + consumerSettings.Server = TServerSetting{"localhost", Server.GrpcPort}; + consumerSettings.UseV2RetryPolicyInCompatMode = true; + return consumerSettings; + } + + TLog& GetLog() { + return Log; + } + + + template <class TConsumerOrProducer> + void Start(const THolder<TConsumerOrProducer>& obj) { + auto startFuture = obj->Start(); + const auto& initResponse = startFuture.GetValueSync(); + UNIT_ASSERT_C(!initResponse.Response.HasError(), "Failed to start: " << initResponse.Response); + } + + THolder<IConsumer> StartConsumer(const TConsumerSettings& settings) { + THolder<IConsumer> consumer = GetPQLib()->CreateConsumer(settings); + Start(consumer); + return consumer; + } + + THolder<IConsumer> StartConsumer() { + return StartConsumer(GetConsumerSettings()); + } + + THolder<IProducer> StartProducer(const TProducerSettings& settings) { + THolder<IProducer> producer = GetPQLib()->CreateProducer(settings); + Start(producer); + return producer; + } + + THolder<IProducer> StartProducer() { + return StartProducer(GetProducerSettings()); + } + + void WriteToTopic(const TVector<TString>& data, IProducer* producer = nullptr) { + THolder<IProducer> localProducer; + if (!producer) { + localProducer = StartProducer(); + producer = localProducer.Get(); + } + TVector<NThreading::TFuture<TProducerCommitResponse>> resps; + for (const TString& d : data) { + Log << TLOG_INFO << "WriteToTopic: " << d; + resps.push_back(producer->Write(d)); + } + for (NThreading::TFuture<TProducerCommitResponse>& r : resps) { + UNIT_ASSERT_C(!r.GetValueSync().Response.HasError(), r.GetValueSync().Response.GetError()); + } + } + + // Read set of sequences from topic + void ReadFromTopic(const TVector<TVector<TString>>& data, bool commit = true, IConsumer* consumer = nullptr) { + THolder<IConsumer> localConsumer; + if (!consumer) { + localConsumer = StartConsumer(); + consumer = localConsumer.Get(); + } + TVector<size_t> positions(data.size()); // Initialy zeroes. + + int wholeCount = 0; + for (const TVector<TString>& seq : data) { + wholeCount += seq.size(); + } + + TSet<ui64> cookies; + + auto processCommit = [&](const TConsumerMessage& resp) { + Log << TLOG_INFO << "ReadFromTopic. Committed: " << resp.Response.GetCommit(); + for (ui64 cookie : resp.Response.GetCommit().GetCookie()) { + UNIT_ASSERT_VALUES_EQUAL(cookies.erase(cookie), 1); + } + }; + + while (wholeCount > 0) { + auto event = consumer->GetNextMessage(); + const auto& resp = event.GetValueSync(); + UNIT_ASSERT_C(!resp.Response.HasError(), resp.Response); + if (!resp.Response.HasData()) { + if (resp.Response.HasCommit()) { + processCommit(resp); + } + continue; + } + Log << TLOG_INFO << "ReadFromTopic. Data: " << resp.Response.GetData(); + UNIT_ASSERT(cookies.insert(resp.Response.GetData().GetCookie()).second); + for (const auto& batch : resp.Response.GetData().GetMessageBatch()) { + // find proper sequence + const TString& firstData = batch.GetMessage(0).GetData(); + size_t seqIndex = 0; + for (; seqIndex < positions.size(); ++seqIndex) { + if (positions[seqIndex] >= data[seqIndex].size()) { // Already seen. + continue; + } + size_t& seqPos = positions[seqIndex]; + const TString& expectedData = data[seqIndex][seqPos]; + if (expectedData == firstData) { + UNIT_ASSERT(batch.MessageSize() <= data[seqIndex].size() - positions[seqIndex]); + ++seqPos; + --wholeCount; + // Match. + for (size_t msgIndex = 1; msgIndex < batch.MessageSize(); ++msgIndex, ++seqPos, --wholeCount) { + UNIT_ASSERT_STRINGS_EQUAL(batch.GetMessage(msgIndex).GetData(), data[seqIndex][seqPos]); + } + break; + } + } + UNIT_ASSERT_LT_C(seqIndex, positions.size(), resp.Response); + } + + if (commit) { + consumer->Commit({resp.Response.GetData().GetCookie()}); + } + } + while (commit && !cookies.empty()) { + auto event = consumer->GetNextMessage(); + const auto& resp = event.GetValueSync(); + UNIT_ASSERT_C(!resp.Response.HasError(), resp.Response); + if (!resp.Response.HasCommit()) { + continue; + } + processCommit(resp); + } + UNIT_ASSERT_VALUES_EQUAL(wholeCount, 0); + } + + void SetSingleDataCenter(const TString& name = "dc1") { + UNIT_ASSERT( + DataCenters.insert(std::make_pair( + name, + NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true} + )).second + ); + LocalDC = name; + } + + void AddDataCenter(const TString& name, const TString& address, bool enabled = true, bool setSelfAsDc = true) { + if (DataCenters.empty() && setSelfAsDc) { + SetSingleDataCenter(); + } + NKikimr::NPersQueueTests::TPQTestClusterInfo info{ + address, + enabled + }; + UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second); + } + + void AddDataCenter(const TString& name, const SDKTestSetup& cluster, bool enabled = true, bool setSelfAsDc = true) { + AddDataCenter(name, TStringBuilder() << "localhost:" << cluster.Server.GrpcPort, enabled, setSelfAsDc); + } + + void EnableDataCenter(const TString& name) { + auto iter = DataCenters.find(name); + UNIT_ASSERT(iter != DataCenters.end()); + Server.AnnoyingClient->UpdateDcEnabled(name, true); + } + void DisableDataCenter(const TString& name) { + auto iter = DataCenters.find(name); + UNIT_ASSERT(iter != DataCenters.end()); + Server.AnnoyingClient->UpdateDcEnabled(name, false); + } + + void ShutdownGRpc() { + Server.ShutdownGRpc(); + } + + void EnableGRpc() { + Server.EnableGRpc(); + Server.WaitInit(GetTestTopic()); + } + + void KickTablets() { + for (ui32 i = 0; i < Server.CleverServer->StaticNodes(); i++) { + Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false); + Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i); + } + } + void AllowTablets() { + for (ui32 i = 0; i < Server.CleverServer->StaticNodes(); i++) { + Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true); + } + } + + void CreateTopic(const TString& topic, const TString& cluster, size_t partitionsCount = 1) { + Server.AnnoyingClient->CreateTopicNoLegacy(BuildFullTopicName(topic, cluster), partitionsCount); + } + + void KillPqrb(const TString& topic, const TString& cluster) { + auto describeResult = Server.AnnoyingClient->Ls(TStringBuilder() << "/Root/PQ/" << BuildFullTopicName(topic, cluster)); + UNIT_ASSERT_C(describeResult->Record.GetPathDescription().HasPersQueueGroup(), describeResult->Record); + Server.AnnoyingClient->KillTablet(*Server.CleverServer, describeResult->Record.GetPathDescription().GetPersQueueGroup().GetBalancerTabletID()); + } +}; +} diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp new file mode 100644 index 0000000000..f5d0b63cb3 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.cpp @@ -0,0 +1,95 @@ +#include "test_pqlib.h" + +namespace NPersQueue::NTests { + + std::tuple<THolder<IProducer>, TProducerCreateResponse> TTestPQLib::CreateProducer( + const TProducerSettings& settings, bool deprecated + ) { + auto producer = PQLib->CreateProducer(settings, Logger, deprecated); + TProducerCreateResponse response = producer->Start().GetValueSync(); + return std::tuple<THolder<IProducer>, TProducerCreateResponse>(std::move(producer), response); + } + + std::tuple<THolder<IProducer>, TProducerCreateResponse> TTestPQLib::CreateProducer( + const TString& topic, + const TString& sourceId, + std::optional<ui32> partitionGroup, + std::optional<NPersQueueCommon::ECodec> codec, + std::optional<bool> reconnectOnFailure, + bool deprecated + ) { + TProducerSettings s; + s.Server = TServerSetting{"localhost", Server.GrpcPort}; + s.Topic = topic; + s.SourceId = sourceId; + if (partitionGroup) { + s.PartitionGroup = partitionGroup.value(); + } + if (codec) { + s.Codec = codec.value(); + } + if (reconnectOnFailure) { + s.ReconnectOnFailure = reconnectOnFailure.value(); + } + return CreateProducer(s, deprecated); + } + + std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateConsumer( + const TConsumerSettings& settings + ) { + auto consumer = PQLib->CreateConsumer(settings, Logger); + TConsumerCreateResponse response = consumer->Start().GetValueSync(); + return std::tuple<THolder<IConsumer>, TConsumerCreateResponse>(std::move(consumer), response); + } + + std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateDeprecatedConsumer( + const TConsumerSettings& settings + ) { + auto consumer = PQLib->CreateConsumer(settings, Logger, true); + TConsumerCreateResponse response = consumer->Start().GetValueSync(); + return std::tuple<THolder<IConsumer>, TConsumerCreateResponse>(std::move(consumer), response); + } + + TConsumerSettings TTestPQLib::MakeSettings( + const TVector<TString>& topics, + const TString& consumerName + ) { + TConsumerSettings s; + s.Server = TServerSetting{"localhost", Server.GrpcPort}; + s.ClientId = consumerName; + s.Topics = topics; + return s; + } + + std::tuple<THolder<IConsumer>, TConsumerCreateResponse> TTestPQLib::CreateConsumer( + const TVector<TString>& topics, + const TString& consumerName, + std::optional<ui32> maxCount, + std::optional<bool> useLockSession, + std::optional<bool> readMirroredPartitions, + std::optional<bool> unpack, + std::optional<ui32> maxInflyRequests, + std::optional<ui32> maxMemoryUsage + ) { + auto s = MakeSettings(topics, consumerName); + if (maxCount) { + s.MaxCount = maxCount.value(); + } + if (unpack) { + s.Unpack = unpack.value(); + } + if (readMirroredPartitions) { + s.ReadMirroredPartitions = readMirroredPartitions.value(); + } + if (maxInflyRequests) { + s.MaxInflyRequests = maxInflyRequests.value(); + } + if (maxMemoryUsage) { + s.MaxMemoryUsage = maxMemoryUsage.value(); + } + if (useLockSession) { + s.UseLockSession = useLockSession.value(); + } + return CreateConsumer(s); + } +} // namespace NPersQueue::NTests
\ No newline at end of file diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h new file mode 100644 index 0000000000..8e6fb6832d --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_pqlib.h @@ -0,0 +1,43 @@ +#pragma once +#include "test_server.h" + +namespace NPersQueue::NTests { + +class TTestPQLib { +public: + TTestPQLib(TTestServer& server) + : Logger(new TCerrLogger(DEBUG_LOG_LEVEL)) + , Server(server) + { + TPQLibSettings settings; + settings.DefaultLogger = Logger; + PQLib = MakeHolder<TPQLib>(settings); + } + + std::tuple<THolder<IProducer>, TProducerCreateResponse> CreateProducer( + const TProducerSettings& settings, bool deprecated + ); + + std::tuple<THolder<IProducer>, TProducerCreateResponse> CreateProducer( + const TString& topic, const TString& sourceId, std::optional<ui32> partitionGroup = {}, + std::optional<NPersQueueCommon::ECodec> codec = ECodec::RAW, std::optional<bool> reconnectOnFailure = {}, + bool deprecated = false); + + std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateConsumer(const TConsumerSettings& settings); + + std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateDeprecatedConsumer(const TConsumerSettings& settings); + + TConsumerSettings MakeSettings(const TVector<TString>& topics, const TString& consumerName = "user"); + + std::tuple<THolder<IConsumer>, TConsumerCreateResponse> CreateConsumer( + const TVector<TString>& topics, const TString& consumerName = "user", std::optional<ui32> maxCount = {}, + std::optional<bool> useLockSession = {}, std::optional<bool> readMirroredPartitions = {}, + std::optional<bool> unpack = {}, std::optional<ui32> maxInflyRequests = {}, + std::optional<ui32> maxMemoryUsage = {}); + +private: + TIntrusivePtr<TCerrLogger> Logger; + TTestServer& Server; + THolder<TPQLib> PQLib; +}; +} // namespace NPersQueue::NTests
\ No newline at end of file diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp new file mode 100644 index 0000000000..6cd1e8094a --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.cpp @@ -0,0 +1,35 @@ +#include "test_server.h" + +#include <ydb/core/client/server/msgbus_server_pq_metacache.h> +#include <kikimr/yndx/grpc_services/persqueue/persqueue.h> +#include <kikimr/yndx/persqueue/msgbus_server/read_session_info.h> + +namespace NPersQueue { + +const TVector<NKikimrServices::EServiceKikimr> TTestServer::LOGGED_SERVICES = { + NKikimrServices::PERSQUEUE, + NKikimrServices::PQ_METACACHE, + NKikimrServices::PQ_READ_PROXY, + NKikimrServices::PQ_WRITE_PROXY, + NKikimrServices::PQ_MIRRORER, +}; + +void TTestServer::PatchServerSettings() { + ServerSettings.RegisterGrpcService<NKikimr::NGRpcService::TGRpcPersQueueService>( + "pq", + NKikimr::NMsgBusProxy::CreatePersQueueMetaCacheV2Id() + ); + ServerSettings.SetPersQueueGetReadSessionsInfoWorkerFactory( + std::make_shared<NKikimr::NMsgBusProxy::TPersQueueGetReadSessionsInfoWorkerWithPQv0Factory>() + ); +} + +void TTestServer::StartIfNeeded(bool start) { + if (start) { + StartServer(); + } +} + + + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h new file mode 100644 index 0000000000..090d9aa47f --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_server.h @@ -0,0 +1,129 @@ +#pragma once +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +#include <ydb/core/testlib/test_pq_client.h> +#include <library/cpp/grpc/server/grpc_server.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/tests_data.h> + +#include <util/system/tempfile.h> + +namespace NPersQueue { + +static constexpr int DEBUG_LOG_LEVEL = 7; + +class TTestServer { +public: + TTestServer(bool start = true, TMaybe<TSimpleSharedPtr<TPortManager>> portManager = Nothing()) + : PortManager(portManager.GetOrElse(MakeSimpleShared<TPortManager>())) + , Port(PortManager->GetPort(2134)) + , GrpcPort(PortManager->GetPort(2135)) + , ServerSettings(NKikimr::NPersQueueTests::PQSettings(Port).SetGrpcPort(GrpcPort)) + , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) + , PQLibSettings(TPQLibSettings{ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) }) + , PQLib(new TPQLib(PQLibSettings)) + { + PatchServerSettings(); + StartIfNeeded(start); + } + + TTestServer(const NKikimr::Tests::TServerSettings& settings, bool start = true) + : PortManager(MakeSimpleShared<TPortManager>()) + , Port(PortManager->GetPort(2134)) + , GrpcPort(PortManager->GetPort(2135)) + , ServerSettings(settings) + , GrpcServerOptions(NGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) + , PQLibSettings(TPQLibSettings{ .DefaultLogger = new TCerrLogger(DEBUG_LOG_LEVEL) }) + , PQLib(new TPQLib(PQLibSettings)) + { + ServerSettings.Port = Port; + ServerSettings.SetGrpcPort(GrpcPort); + PatchServerSettings(); + StartIfNeeded(start); + } + + void StartServer(bool doClientInit = true) { + PrepareNetDataFile(); + CleverServer = MakeHolder<NKikimr::Tests::TServer>(ServerSettings); + CleverServer->EnableGRpc(GrpcServerOptions); + AnnoyingClient = MakeHolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient>(ServerSettings, GrpcPort); + EnableLogs(LOGGED_SERVICES); + if (doClientInit) { + AnnoyingClient->FullInit(); + } + } + + void ShutdownGRpc() { + CleverServer->ShutdownGRpc(); + } + + void EnableGRpc() { + CleverServer->EnableGRpc(GrpcServerOptions); + } + + void ShutdownServer() { + CleverServer = nullptr; + } + + void RestartServer() { + ShutdownServer(); + StartServer(); + } + + void EnableLogs(const TVector<NKikimrServices::EServiceKikimr> services, + NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) { + Y_VERIFY(CleverServer != nullptr, "Start server before enabling logs"); + for (auto s : services) { + CleverServer->GetRuntime()->SetLogPriority(s, prio); + } + } + + void WaitInit(const TString& topic) { + TProducerSettings s; + s.Topic = topic; + s.Server = TServerSetting{"localhost", GrpcPort}; + s.SourceId = "src"; + + while (PQLib->CreateProducer(s, {}, false)->Start().GetValueSync().Response.HasError()) { + Sleep(TDuration::MilliSeconds(200)); + } + } + + bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") { + if (NetDataFile) + return false; + NetDataFile = MakeHolder<TTempFileHandle>("netData.tsv"); + NetDataFile->Write(content.Data(), content.Size()); + NetDataFile->FlushData(); + ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); + return true; + } + + void UpdateDC(const TString& name, bool local, bool enabled) { + AnnoyingClient->UpdateDC(name, local, enabled); + } + +private: + void PatchServerSettings(); + void StartIfNeeded(bool start); + +public: + TSimpleSharedPtr<TPortManager> PortManager; + ui16 Port; + ui16 GrpcPort; + + THolder<NKikimr::Tests::TServer> CleverServer; + NKikimr::Tests::TServerSettings ServerSettings; + NGrpc::TServerOptions GrpcServerOptions; + THolder<TTempFileHandle> NetDataFile; + + THolder<NKikimr::NPersQueueTests::TFlatMsgBusPQClient> AnnoyingClient; + + TPQLibSettings PQLibSettings; + THolder<TPQLib> PQLib; + + static const TVector<NKikimrServices::EServiceKikimr> LOGGED_SERVICES; +}; + +} // namespace NPersQueue diff --git a/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h new file mode 100644 index 0000000000..13b4da2772 --- /dev/null +++ b/kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/test_utils.h @@ -0,0 +1,101 @@ +#pragma once +#include <util/generic/ptr.h> +#include <util/generic/size_literals.h> +#include <library/cpp/threading/chunk_queue/queue.h> +#include <util/generic/overloaded.h> +#include <library/cpp/testing/unittest/registar.h> + +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/ut_utils/sdk_test_setup.h> +#include <kikimr/persqueue/sdk/deprecated/cpp/v2/persqueue.h> + +namespace NPersQueue { + +using namespace NThreading; +using namespace NYdb::NPersQueue; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; +//using namespace NPersQueue::V1; + +template <class TPQLibObject> +void DestroyAndWait(THolder<TPQLibObject>& object) { + if (object) { + auto isDead = object->IsDead(); + object = nullptr; + isDead.GetValueSync(); + } +} + +inline bool GrpcV1EnabledByDefault() { + static const bool enabled = std::getenv("PERSQUEUE_GRPC_API_V1_ENABLED"); + return enabled; +} + +class TCallbackCredentialsProvider : public ICredentialsProvider { + std::function<void(NPersQueue::TCredentials*)> Callback; +public: + TCallbackCredentialsProvider(std::function<void(NPersQueue::TCredentials*)> callback) + : Callback(std::move(callback)) + {} + + void FillAuthInfo(NPersQueue::TCredentials* authInfo) const { + Callback(authInfo); + } +}; + +struct TWriteResult { + bool Ok = false; + // No acknowledgement is expected from a writer under test + bool NoWait = false; + TString ResponseDebugString = TString(); +}; + +struct TAcknowledgableMessage { + TString Value; + ui64 SequenceNumber; + TInstant CreatedAt; + TPromise<TWriteResult> AckPromise; +}; + +class IClientEventLoop { +protected: + std::atomic_bool MayStop; + std::atomic_bool MustStop; + bool Stopped = false; + std::unique_ptr<TThread> Thread; + TLog Log; + +public: + IClientEventLoop() + : MayStop() + , MustStop() + , MessageBuffer() + {} + + void AllowStop() { + MayStop = true; + } + + void WaitForStop() { + if (!Stopped) { + Log << TLOG_INFO << "Wait for writer to die on itself"; + Thread->Join(); + Log << TLOG_INFO << "Client write event loop stopped"; + } + Stopped = true; + } + + virtual ~IClientEventLoop() { + MustStop = true; + if (!Stopped) { + Log << TLOG_INFO << "Wait for client write event loop to stop"; + Thread->Join(); + Log << TLOG_INFO << "Client write event loop stopped"; + } + Stopped = true; + } + + TManyOneQueue<TAcknowledgableMessage> MessageBuffer; + +}; + +} // namespace NPersQueue |