Fixed time waits

ПР про избавление от застревающих портов в состоянии TIME\_WAIT
Этот клиент используется в universal\_fetcher-е, который в свою очередь используется для скачивания резалтов с distbuild-а. Соотвественно так как он качает слишком быстро, то на хосте заканчивается кол-во эфемерных портов (потому что порты после закрытых соединений остаются в состоянии TIME\_WAIT). Эту проблему можно решить, если дожидаться закрытия соединения со стороны сервера.

В этом ПР-е сделан карантин connection-ов, который позволяет асинхронно дожидаться закрытия со стороны сервера
commit_hash:80ac8208e4efa8bc9eccbcfe20869899d8e11c5a

1 files changed, 36 insertions, 6 deletions

diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp
index 6fb04755391..87bbabc3f83 100644
--- a/library/cpp/http/simple/http_client.cpp
+++ b/library/cpp/http/simple/http_client.cpp
@@ -6,15 +6,25 @@
 #include <util/string/cast.h>
 #include <util/string/join.h>
 #include <util/string/split.h>
+#include <util/system/spinlock.h>
+#include <library/cpp/cache/cache.h>
+
+
+TSpinLock TKeepAliveHttpClient::ConnectionQuarantineMutex;
+TQueue<THolder<NPrivate::THttpConnection>> TKeepAliveHttpClient::ConnectionQuarantine;
 
 TKeepAliveHttpClient::TKeepAliveHttpClient(const TString& host,
                                            ui32 port,
                                            TDuration socketTimeout,
-                                           TDuration connectTimeout)
+                                           TDuration connectTimeout,
+                                           bool useKeepAlive,
+                                           bool useConnectionPool)
     : Host(CutHttpPrefix(host))
     , Port(port)
     , SocketTimeout(socketTimeout)
     , ConnectTimeout(connectTimeout)
+    , UseKeepAlive(useKeepAlive)
+    , UseConnectionPool(useConnectionPool)
     , IsHttps(host.StartsWith("https"))
     , IsClosingRequired(false)
     , HttpsVerification(TVerifyCert{Host})
@@ -159,13 +169,29 @@ bool TKeepAliveHttpClient::CreateNewConnectionIfNeeded() {
                                                            ConnectTimeout,
                                                            IsHttps,
                                                            ClientCertificate,
-                                                           HttpsVerification);
+                                                           HttpsVerification,
+                                                           UseKeepAlive);
         IsClosingRequired = false;
         return true;
     }
     return false;
 }
 
+TKeepAliveHttpClient::~TKeepAliveHttpClient() {
+    if (UseConnectionPool) {
+        THolder<NPrivate::THttpConnection> oldConnection;
+        with_lock(ConnectionQuarantineMutex) {
+            while (ConnectionQuarantine.size() > 100) {
+                oldConnection = std::move(ConnectionQuarantine.front());
+
+                ConnectionQuarantine.pop();
+                oldConnection.Reset();
+            }
+            ConnectionQuarantine.push(std::move(Connection));
+        }
+    }
+}
+
 THttpRequestException::THttpRequestException(int statusCode)
     : StatusCode(statusCode)
 {
@@ -180,6 +206,8 @@ TSimpleHttpClient::TSimpleHttpClient(const TOptions& options)
     , Port(options.Port())
     , SocketTimeout(options.SocketTimeout())
     , ConnectTimeout(options.ConnectTimeout())
+    , UseKeepAlive(options.UseKeepAlive())
+    , UseConnectionPool(options.UseConnectionPool())
 {
 }
 
@@ -229,7 +257,8 @@ namespace NPrivate {
                                      TDuration connTimeout,
                                      bool isHttps,
                                      const TMaybe<TOpenSslClientIO::TOptions::TClientCert>& clientCert,
-                                     const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert)
+                                     const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert,
+                                     bool keepAlive)
         : Addr(Resolve(host, port))
         , Socket(Connect(Addr, sockTimeout, connTimeout, host, port))
         , SocketIn(Socket)
@@ -249,8 +278,9 @@ namespace NPrivate {
         } else {
             HttpOut = MakeHolder<THttpOutput>(&SocketOut);
         }
-
-        HttpOut->EnableKeepAlive(true);
+        if (keepAlive) {
+            HttpOut->EnableKeepAlive(true);
+        }
     }
 
     TNetworkAddress THttpConnection::Resolve(const TString& host, ui32 port) {
@@ -292,7 +322,7 @@ TSimpleHttpClient::~TSimpleHttpClient() {
 }
 
 TKeepAliveHttpClient TSimpleHttpClient::CreateClient() const {
-    TKeepAliveHttpClient cl(Host, Port, SocketTimeout, ConnectTimeout);
+    TKeepAliveHttpClient cl(Host, Port, SocketTimeout, ConnectTimeout, UseKeepAlive, UseConnectionPool);
 
     if (!HttpsVerification) {
         cl.DisableVerificationForHttps();