#pragma once #include #include #include #include #include #include "interconnect_common.h" #include "interconnect_counters.h" #include "interconnect_tcp_session.h" #include "profiler.h" #define ICPROXY_PROFILED TFunction func(*this, __func__, __LINE__) namespace NActors { /* WARNING: all proxy actors should be alive during actorsystem activity */ class TInterconnectProxyTCP : public TActor , public TInterconnectLoggingBase , public TProfiled { enum { EvCleanupEventQueue = EventSpaceBegin(TEvents::ES_PRIVATE), EvQueryStats, EvStats, EvPassAwayIfNeeded, }; struct TEvCleanupEventQueue : TEventLocal {}; public: struct TEvQueryStats : TEventLocal {}; struct TProxyStats { TString Path; TString State; TScopeId PeerScopeId; TInstant LastSessionDieTime; ui64 TotalOutputQueueSize; bool Connected; TString Host; ui16 Port; TInstant LastErrorTimestamp; TString LastErrorKind; TString LastErrorExplanation; TDuration Ping; i64 ClockSkew; TString Encryption; }; struct TEvStats : TEventLocal { ui32 PeerNodeId; TProxyStats ProxyStats; }; static constexpr EActivityType ActorActivityType() { return INTERCONNECT_PROXY_TCP; } TInterconnectProxyTCP(const ui32 node, TInterconnectProxyCommon::TPtr common, IActor **dynamicPtr = nullptr); STFUNC(StateInit) { Bootstrap(); if (ev->Type != TEvents::TSystem::Bootstrap) { // for dynamic nodes we do not receive Bootstrap event Receive(ev, ctx); } } void Bootstrap(); void Registered(TActorSystem* sys, const TActorId& owner) override; private: friend class TInterconnectSessionTCP; friend class TInterconnectSessionTCPv0; friend class THandshake; friend class TInputSessionTCP; void UnregisterSession(TInterconnectSessionTCP* session); #define SESSION_EVENTS(HANDLER) \ fFunc(TEvInterconnect::EvForward, HANDLER) \ fFunc(TEvInterconnect::TEvConnectNode::EventType, HANDLER) \ fFunc(TEvents::TEvSubscribe::EventType, HANDLER) \ fFunc(TEvents::TEvUnsubscribe::EventType, HANDLER) #define INCOMING_HANDSHAKE_EVENTS(HANDLER) \ fFunc(TEvHandshakeAsk::EventType, HANDLER) \ fFunc(TEvHandshakeRequest::EventType, HANDLER) #define HANDSHAKE_STATUS_EVENTS(HANDLER) \ hFunc(TEvHandshakeDone, HANDLER) \ hFunc(TEvHandshakeFail, HANDLER) #define PROXY_STFUNC(STATE, SESSION_HANDLER, INCOMING_HANDSHAKE_HANDLER, \ HANDSHAKE_STATUS_HANDLER, DISCONNECT_HANDLER, \ WAKEUP_HANDLER, NODE_INFO_HANDLER) \ STATEFN(STATE) { \ const ui32 type = ev->GetTypeRewrite(); \ const bool profiled = type != TEvInterconnect::EvForward \ && type != TEvInterconnect::EvConnectNode \ && type != TEvents::TSystem::Subscribe \ && type != TEvents::TSystem::Unsubscribe; \ if (profiled) { \ TProfiled::Start(); \ } \ { \ TProfiled::TFunction func(*this, __func__, __LINE__); \ switch (type) { \ SESSION_EVENTS(SESSION_HANDLER) \ INCOMING_HANDSHAKE_EVENTS(INCOMING_HANDSHAKE_HANDLER) \ HANDSHAKE_STATUS_EVENTS(HANDSHAKE_STATUS_HANDLER) \ cFunc(TEvInterconnect::EvDisconnect, DISCONNECT_HANDLER) \ hFunc(TEvents::TEvWakeup, WAKEUP_HANDLER) \ hFunc(TEvGetSecureSocket, Handle) \ hFunc(NMon::TEvHttpInfo, GenerateHttpInfo) \ cFunc(EvCleanupEventQueue, HandleCleanupEventQueue) \ hFunc(TEvInterconnect::TEvNodeInfo, NODE_INFO_HANDLER) \ cFunc(TEvInterconnect::EvClosePeerSocket, HandleClosePeerSocket) \ cFunc(TEvInterconnect::EvCloseInputSession, HandleCloseInputSession) \ cFunc(TEvInterconnect::EvPoisonSession, HandlePoisonSession) \ hFunc(TEvSessionBufferSizeRequest, HandleSessionBufferSizeRequest) \ hFunc(TEvQueryStats, Handle) \ cFunc(TEvInterconnect::EvTerminate, HandleTerminate) \ cFunc(EvPassAwayIfNeeded, HandlePassAwayIfNeeded) \ default: \ Y_FAIL("unexpected event Type# 0x%08" PRIx32, type); \ } \ } \ if (profiled) { \ if (TProfiled::Duration() >= TDuration::MilliSeconds(16)) { \ const TString report = TProfiled::Format(); \ LOG_ERROR_IC("ICP35", "event processing took too much time %s", report.data()); \ } \ TProfiled::Finish(); \ } \ } template void Ignore(T& /*ev*/) { ICPROXY_PROFILED; } void Ignore() { ICPROXY_PROFILED; } void Ignore(TEvHandshakeDone::TPtr& ev) { ICPROXY_PROFILED; Y_VERIFY(ev->Sender != IncomingHandshakeActor); Y_VERIFY(ev->Sender != OutgoingHandshakeActor); } void Ignore(TEvHandshakeFail::TPtr& ev) { ICPROXY_PROFILED; Y_VERIFY(ev->Sender != IncomingHandshakeActor); Y_VERIFY(ev->Sender != OutgoingHandshakeActor); LogHandshakeFail(ev, true); } const char* State = nullptr; TInstant StateSwitchTime; template void SwitchToState(int line, const char* name, TArgs&&... args) { ICPROXY_PROFILED; LOG_DEBUG_IC("ICP77", "@%d %s -> %s", line, State, name); State = name; StateSwitchTime = TActivationContext::Now(); Become(std::forward(args)...); Y_VERIFY(!Terminated || CurrentStateFunc() == &TThis::HoldByError); // ensure we never escape this state if (CurrentStateFunc() != &TThis::PendingActivation) { PassAwayTimestamp = TInstant::Max(); } } TInstant PassAwayTimestamp; bool PassAwayScheduled = false; void SwitchToInitialState() { ICPROXY_PROFILED; Y_VERIFY(!PendingSessionEvents && !PendingIncomingHandshakeEvents, "%s PendingSessionEvents# %zu" " PendingIncomingHandshakeEvents# %zu State# %s", LogPrefix.data(), PendingSessionEvents.size(), PendingIncomingHandshakeEvents.size(), State); SwitchToState(__LINE__, "PendingActivation", &TThis::PendingActivation); if (DynamicPtr && !PassAwayScheduled && PassAwayTimestamp != TInstant::Max()) { TActivationContext::Schedule(PassAwayTimestamp, new IEventHandle(EvPassAwayIfNeeded, 0, SelfId(), {}, nullptr, 0)); PassAwayScheduled = true; } } void HandlePassAwayIfNeeded() { Y_VERIFY(PassAwayScheduled); if (PassAwayTimestamp != TInstant::Max()) { PassAway(); } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // PendingActivation // // In this state we are just waiting for some activities, which may include: // * an external Session event // * incoming handshake request // // Upon receiving such event, we put it to corresponding queue and initiate start up by calling IssueGetNodeRequest, // which, as the name says, issued TEvGetNode to the nameservice and arms timer to handle timeout (which should not // occur, but we want to be sure we don't hang on this), and then switches to PendingNodeInfo state. PROXY_STFUNC(PendingActivation, RequestNodeInfo, // Session events RequestNodeInfoForIncomingHandshake, // Incoming handshake requests Ignore, // Handshake status Ignore, // Disconnect request Ignore, // Wakeup Ignore // Node info ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // PendingNodeInfo // // This state is entered when we asked nameserver to provide description for peer node we are working with. All // external Session events and incoming handshake requests are enqueued into their respective queues, TEvNodeInfo // is main event that triggers processing. On success, we try to initiate outgoing handshake if needed, or process // incoming handshakes. On error, we enter HoldByError state. // // NOTE: handshake status events are also enqueued as the handshake actor may have generated failure event due to // timeout or some other reason without waiting for acknowledge, and it must be processed correctly to prevent // session hang PROXY_STFUNC(PendingNodeInfo, EnqueueSessionEvent, // Session events EnqueueIncomingHandshakeEvent, // Incoming handshake requests EnqueueIncomingHandshakeEvent, // Handshake status Disconnect, // Disconnect request ConfigureTimeout, // Wakeup Configure // Node info ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // PendingConnection // // Here we have issued outgoing handshake or have accepted (or may be both) incoming handshake and we are waiting for // the status of the handshake. When one if handshakes finishes, we use this status to establish connection (or to // go to error state). When one handshake terminates with error while other is running, we will still wait for the // second one to finish. PROXY_STFUNC(PendingConnection, EnqueueSessionEvent, // Session events IncomingHandshake, // Incoming handshake requests HandleHandshakeStatus, // Handshake status Disconnect, // Disconnect request Ignore, // Wakeup Ignore // Node info ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // StateWork // // We have accepted session and process any incoming messages with the session. Incoming handshakes are accepted // concurrently and applied when finished. PROXY_STFUNC(StateWork, ForwardSessionEventToSession, // Session events IncomingHandshake, // Incoming handshake requests HandleHandshakeStatus, // Handshake status Disconnect, // Disconnect request Ignore, // Wakeup Ignore // Node info ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // HoldByError // // When something bad happens with the connection, we sleep in this state. After wake up we go back to // PendingActivation. PROXY_STFUNC(HoldByError, DropSessionEvent, // Session events RequestNodeInfoForIncomingHandshake, // Incoming handshake requests Ignore, // Handshake status Ignore, // Disconnect request WakeupFromErrorState, // Wakeup Ignore // Node info ) #undef SESSION_EVENTS #undef INCOMING_HANDSHAKE_EVENTS #undef HANDSHAKE_STATUS_EVENTS #undef PROXY_STFUNC void ForwardSessionEventToSession(STATEFN_SIG); void EnqueueSessionEvent(STATEFN_SIG); // Incoming handshake handlers, including special wrapper when the IncomingHandshake is used as fFunc void IncomingHandshake(STATEFN_SIG) { switch (ev->GetTypeRewrite()) { hFunc(TEvHandshakeAsk, IncomingHandshake); hFunc(TEvHandshakeRequest, IncomingHandshake); default: Y_FAIL(); } } void IncomingHandshake(TEvHandshakeAsk::TPtr& ev); void IncomingHandshake(TEvHandshakeRequest::TPtr& ev); void RequestNodeInfo(STATEFN_SIG); void RequestNodeInfoForIncomingHandshake(STATEFN_SIG); void StartInitialHandshake(); void StartResumeHandshake(ui64 inputCounter); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Incoming handshake event queue processing //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void EnqueueIncomingHandshakeEvent(STATEFN_SIG); void EnqueueIncomingHandshakeEvent(TEvHandshakeDone::TPtr& ev); void EnqueueIncomingHandshakeEvent(TEvHandshakeFail::TPtr& ev); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // PendingNodeInfo //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// IEventBase* ConfigureTimeoutCookie; // pointer to the scheduled event used to match sent and received events void StartConfiguring(); void Configure(TEvInterconnect::TEvNodeInfo::TPtr& ev); void ConfigureTimeout(TEvents::TEvWakeup::TPtr& ev); void ProcessConfigured(); void HandleHandshakeStatus(TEvHandshakeDone::TPtr& ev); void HandleHandshakeStatus(TEvHandshakeFail::TPtr& ev); void TransitToErrorState(TString Explanation, bool updateErrorLog = true); void WakeupFromErrorState(TEvents::TEvWakeup::TPtr& ev); void Disconnect(); const ui32 PeerNodeId; IActor **DynamicPtr; void ValidateEvent(TAutoPtr& ev, const char* func) { if (SelfId().NodeId() == PeerNodeId) { TString msg = Sprintf("Event Type# 0x%08" PRIx32 " TypeRewrite# 0x%08" PRIx32 " from Sender# %s sent to the proxy for the node itself via Interconnect;" " THIS IS NOT A BUG IN INTERCONNECT, check the event sender instead", ev->Type, ev->GetTypeRewrite(), ev->Sender.ToString().data()); LOG_ERROR_IC("ICP03", "%s", msg.data()); Y_VERIFY_DEBUG(false, "%s", msg.data()); } Y_VERIFY(ev->GetTypeRewrite() != TEvInterconnect::EvForward || ev->Recipient.NodeId() == PeerNodeId, "Recipient/Proxy NodeId mismatch Recipient# %s Type# 0x%08" PRIx32 " PeerNodeId# %" PRIu32 " Func# %s", ev->Recipient.ToString().data(), ev->Type, PeerNodeId, func); } // Common with helpers // All proxy actors share the same information in the object // read only TInterconnectProxyCommon::TPtr const Common; const TActorId& GetNameserviceId() const { return Common->NameserviceId; } TString TechnicalPeerHostName; std::shared_ptr Metrics; void HandleClosePeerSocket(); void HandleCloseInputSession(); void HandlePoisonSession(); void HandleSessionBufferSizeRequest(TEvSessionBufferSizeRequest::TPtr& ev); bool CleanupEventQueueScheduled = false; void ScheduleCleanupEventQueue(); void HandleCleanupEventQueue(); void CleanupEventQueue(); // hold all events before connection is established struct TPendingSessionEvent { TInstant Deadline; ui32 Size; THolder Event; TPendingSessionEvent(TInstant deadline, ui32 size, TAutoPtr event) : Deadline(deadline) , Size(size) , Event(event) {} }; TDeque PendingSessionEvents; ui64 PendingSessionEventsSize = 0; void ProcessPendingSessionEvents(); void DropSessionEvent(STATEFN_SIG); TInterconnectSessionTCP* Session = nullptr; TActorId SessionID; // virtual ids used during handshake to check if it is the connection // for the same session or to find out the latest shandshake // it's virtual because session actor apears after successfull handshake TActorId SessionVirtualId; TActorId RemoteSessionVirtualId; TActorId GenerateSessionVirtualId() { ICPROXY_PROFILED; const ui64 localId = TlsActivationContext->ExecutorThread.ActorSystem->AllocateIDSpace(1); return NActors::TActorId(SelfId().NodeId(), 0, localId, 0); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// TActorId IncomingHandshakeActor; TInstant IncomingHandshakeActorFilledIn; TInstant IncomingHandshakeActorReset; TMaybe LastSerialFromIncomingHandshake; THolder HeldHandshakeReply; void DropIncomingHandshake(bool poison = true) { ICPROXY_PROFILED; if (const TActorId& actorId = std::exchange(IncomingHandshakeActor, TActorId())) { LOG_DEBUG_IC("ICP111", "dropped incoming handshake: %s poison: %s", actorId.ToString().data(), poison ? "true" : "false"); if (poison) { Send(actorId, new TEvents::TEvPoisonPill); } LastSerialFromIncomingHandshake.Clear(); HeldHandshakeReply.Reset(); IncomingHandshakeActorReset = TActivationContext::Now(); } } void DropOutgoingHandshake(bool poison = true) { ICPROXY_PROFILED; if (const TActorId& actorId = std::exchange(OutgoingHandshakeActor, TActorId())) { LOG_DEBUG_IC("ICP112", "dropped outgoing handshake: %s poison: %s", actorId.ToString().data(), poison ? "true" : "false"); if (poison) { Send(actorId, new TEvents::TEvPoisonPill); } OutgoingHandshakeActorReset = TActivationContext::Now(); } } void DropHandshakes() { ICPROXY_PROFILED; DropIncomingHandshake(); DropOutgoingHandshake(); } void PrepareNewSessionHandshake() { ICPROXY_PROFILED; // drop existing session if we have one if (Session) { LOG_INFO_IC("ICP04", "terminating current session as we are negotiating a new one"); IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::Terminate, TDisconnectReason::NewSession()); } // ensure we have no current session Y_VERIFY(!Session); // switch to pending connection state -- we wait for handshakes, we want more handshakes! SwitchToState(__LINE__, "PendingConnection", &TThis::PendingConnection); } void IssueIncomingHandshakeReply(const TActorId& handshakeId, ui64 peerLocalId, THolder event); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// TActorId OutgoingHandshakeActor; TInstant OutgoingHandshakeActorCreated; TInstant OutgoingHandshakeActorReset; TInstant LastSessionDieTime; void GenerateHttpInfo(NMon::TEvHttpInfo::TPtr& ev); void Handle(TEvQueryStats::TPtr& ev); TDuration HoldByErrorWakeupDuration = TDuration::Zero(); TEvents::TEvWakeup* HoldByErrorWakeupCookie; THolder RemoteProgramInfo; NInterconnect::TSecureSocketContext::TPtr SecureContext; void Handle(TEvGetSecureSocket::TPtr ev) { auto socket = MakeIntrusive(*ev->Get()->Socket, SecureContext); Send(ev->Sender, new TEvSecureSocket(std::move(socket))); } TDeque> PendingIncomingHandshakeEvents; TDeque> ErrorStateLog; void UpdateErrorStateLog(TInstant now, TString kind, TString explanation) { ICPROXY_PROFILED; if (ErrorStateLog) { auto& back = ErrorStateLog.back(); TString lastKind, lastExpl; if (kind == std::get<1>(back) && explanation == std::get<2>(back)) { std::get<0>(back) = now; ++std::get<3>(back); return; } } ErrorStateLog.emplace_back(now, std::move(kind), std::move(explanation), 1); if (ErrorStateLog.size() > 20) { ErrorStateLog.pop_front(); } } void LogHandshakeFail(TEvHandshakeFail::TPtr& ev, bool inconclusive); bool Terminated = false; void HandleTerminate(); void PassAway() override; }; }