Fixed time waits

ПР про избавление от застревающих портов в состоянии TIME\_WAIT
Этот клиент используется в universal\_fetcher-е, который в свою очередь используется для скачивания резалтов с distbuild-а. Соотвественно так как он качает слишком быстро, то на хосте заканчивается кол-во эфемерных портов (потому что порты после закрытых соединений остаются в состоянии TIME\_WAIT). Эту проблему можно решить, если дожидаться закрытия соединения со стороны сервера.

В этом ПР-е сделан карантин connection-ов, который позволяет асинхронно дожидаться закрытия со стороны сервера
commit_hash:80ac8208e4efa8bc9eccbcfe20869899d8e11c5a

4 files changed, 74 insertions, 8 deletions

diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp
index 6fb04755391..87bbabc3f83 100644
--- a/library/cpp/http/simple/http_client.cpp
+++ b/library/cpp/http/simple/http_client.cpp
@@ -6,15 +6,25 @@
 #include <util/string/cast.h>
 #include <util/string/join.h>
 #include <util/string/split.h>
+#include <util/system/spinlock.h>
+#include <library/cpp/cache/cache.h>
+
+
+TSpinLock TKeepAliveHttpClient::ConnectionQuarantineMutex;
+TQueue<THolder<NPrivate::THttpConnection>> TKeepAliveHttpClient::ConnectionQuarantine;
 
 TKeepAliveHttpClient::TKeepAliveHttpClient(const TString& host,
                                            ui32 port,
                                            TDuration socketTimeout,
-                                           TDuration connectTimeout)
+                                           TDuration connectTimeout,
+                                           bool useKeepAlive,
+                                           bool useConnectionPool)
     : Host(CutHttpPrefix(host))
     , Port(port)
     , SocketTimeout(socketTimeout)
     , ConnectTimeout(connectTimeout)
+    , UseKeepAlive(useKeepAlive)
+    , UseConnectionPool(useConnectionPool)
     , IsHttps(host.StartsWith("https"))
     , IsClosingRequired(false)
     , HttpsVerification(TVerifyCert{Host})
@@ -159,13 +169,29 @@ bool TKeepAliveHttpClient::CreateNewConnectionIfNeeded() {
                                                            ConnectTimeout,
                                                            IsHttps,
                                                            ClientCertificate,
-                                                           HttpsVerification);
+                                                           HttpsVerification,
+                                                           UseKeepAlive);
         IsClosingRequired = false;
         return true;
     }
     return false;
 }
 
+TKeepAliveHttpClient::~TKeepAliveHttpClient() {
+    if (UseConnectionPool) {
+        THolder<NPrivate::THttpConnection> oldConnection;
+        with_lock(ConnectionQuarantineMutex) {
+            while (ConnectionQuarantine.size() > 100) {
+                oldConnection = std::move(ConnectionQuarantine.front());
+
+                ConnectionQuarantine.pop();
+                oldConnection.Reset();
+            }
+            ConnectionQuarantine.push(std::move(Connection));
+        }
+    }
+}
+
 THttpRequestException::THttpRequestException(int statusCode)
     : StatusCode(statusCode)
 {
@@ -180,6 +206,8 @@ TSimpleHttpClient::TSimpleHttpClient(const TOptions& options)
     , Port(options.Port())
     , SocketTimeout(options.SocketTimeout())
     , ConnectTimeout(options.ConnectTimeout())
+    , UseKeepAlive(options.UseKeepAlive())
+    , UseConnectionPool(options.UseConnectionPool())
 {
 }
 
@@ -229,7 +257,8 @@ namespace NPrivate {
                                      TDuration connTimeout,
                                      bool isHttps,
                                      const TMaybe<TOpenSslClientIO::TOptions::TClientCert>& clientCert,
-                                     const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert)
+                                     const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert,
+                                     bool keepAlive)
         : Addr(Resolve(host, port))
         , Socket(Connect(Addr, sockTimeout, connTimeout, host, port))
         , SocketIn(Socket)
@@ -249,8 +278,9 @@ namespace NPrivate {
         } else {
             HttpOut = MakeHolder<THttpOutput>(&SocketOut);
         }
-
-        HttpOut->EnableKeepAlive(true);
+        if (keepAlive) {
+            HttpOut->EnableKeepAlive(true);
+        }
     }
 
     TNetworkAddress THttpConnection::Resolve(const TString& host, ui32 port) {
@@ -292,7 +322,7 @@ TSimpleHttpClient::~TSimpleHttpClient() {
 }
 
 TKeepAliveHttpClient TSimpleHttpClient::CreateClient() const {
-    TKeepAliveHttpClient cl(Host, Port, SocketTimeout, ConnectTimeout);
+    TKeepAliveHttpClient cl(Host, Port, SocketTimeout, ConnectTimeout, UseKeepAlive, UseConnectionPool);
 
     if (!HttpsVerification) {
         cl.DisableVerificationForHttps();
diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h
index 3860862698d..87d3dc095af 100644
--- a/library/cpp/http/simple/http_client.h
+++ b/library/cpp/http/simple/http_client.h
@@ -8,6 +8,8 @@
 #include <util/generic/strbuf.h>
 #include <util/generic/yexception.h>
 #include <util/network/socket.h>
+#include <util/generic/queue.h>
+#include <util/system/spinlock.h>
 
 #include <library/cpp/http/io/stream.h>
 #include <library/cpp/http/misc/httpcodes.h>
@@ -50,7 +52,12 @@ public:
     TKeepAliveHttpClient(const TString& host,
                          ui32 port,
                          TDuration socketTimeout = TDuration::Seconds(5),
-                         TDuration connectTimeout = TDuration::Seconds(30));
+                         TDuration connectTimeout = TDuration::Seconds(30),
+                         bool useKeepAlive = true,
+                         bool useConnectionPool = false);
+
+    TKeepAliveHttpClient(TKeepAliveHttpClient&&) = default;
+    ~TKeepAliveHttpClient();
 
     THttpCode DoGet(const TStringBuf relativeUrl,
                     IOutputStream* output = nullptr,
@@ -117,8 +124,13 @@ private:
     const ui32 Port;
     const TDuration SocketTimeout;
     const TDuration ConnectTimeout;
+    const bool UseKeepAlive;
+    const bool UseConnectionPool;
     const bool IsHttps;
 
+    static TSpinLock ConnectionQuarantineMutex;
+    static TQueue<THolder<NPrivate::THttpConnection>> ConnectionQuarantine;
+
     THolder<NPrivate::THttpConnection> Connection;
     bool IsClosingRequired;
     TMaybe<TClientCert> ClientCertificate;
@@ -158,6 +170,8 @@ protected:
     const ui32 Port;
     const TDuration SocketTimeout;
     const TDuration ConnectTimeout;
+    const bool UseKeepAlive = true;
+    const bool UseConnectionPool = false;
     bool HttpsVerification = false;
 
 public:
@@ -215,7 +229,8 @@ namespace NPrivate {
                         TDuration connTimeout,
                         bool isHttps,
                         const TMaybe<TOpenSslClientIO::TOptions::TClientCert>& clientCert,
-                        const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert);
+                        const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert,
+                        bool keepAlive = true);
 
         bool IsOk() const {
             return IsNotSocketClosedByOtherSide(Socket);
diff --git a/library/cpp/http/simple/http_client_options.h b/library/cpp/http/simple/http_client_options.h
index 58849556a91..f237a3770a9 100644
--- a/library/cpp/http/simple/http_client_options.h
+++ b/library/cpp/http/simple/http_client_options.h
@@ -61,10 +61,30 @@ public:
         return MaxRedirectCount_;
     }
 
+    TSelf& UseKeepAlive(bool useKeepAlive) {
+        UseKeepAlive_ = useKeepAlive;
+        return *this;
+    }
+
+    bool UseKeepAlive() const noexcept {
+        return UseKeepAlive_;
+    }
+
+    TSelf& UseConnectionPool(bool useConnectionPool) {
+        UseConnectionPool_ = useConnectionPool;
+        return *this;
+    }
+
+    bool UseConnectionPool() const noexcept {
+        return UseConnectionPool_;
+    }
+
 private:
     TString Host_;
     ui16 Port_;
     TDuration SocketTimeout_ = TDuration::Seconds(5);
     TDuration ConnectTimeout_ = TDuration::Seconds(30);
     int MaxRedirectCount_ = INT_MAX;
+    bool UseKeepAlive_ = true;
+    bool UseConnectionPool_ = false;
 };
diff --git a/library/cpp/http/simple/ya.make b/library/cpp/http/simple/ya.make
index 6a4e5775a4d..c645fb38442 100644
--- a/library/cpp/http/simple/ya.make
+++ b/library/cpp/http/simple/ya.make
@@ -1,6 +1,7 @@
 LIBRARY()
 
 PEERDIR(
+    library/cpp/cache
     library/cpp/http/io
     library/cpp/openssl/io
     library/cpp/string_utils/url