Common retry policy for discovery.

7 files changed, 16 insertions, 22 deletions

diff --git a/ydb/library/yql/providers/common/http_gateway/CMakeLists.txt b/ydb/library/yql/providers/common/http_gateway/CMakeLists.txt
index c381059a0ad..9e553621ae1 100644
--- a/ydb/library/yql/providers/common/http_gateway/CMakeLists.txt
+++ b/ydb/library/yql/providers/common/http_gateway/CMakeLists.txt
@@ -24,4 +24,5 @@ target_link_libraries(providers-common-http_gateway PUBLIC
 )
 target_sources(providers-common-http_gateway PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.cpp
 )
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_retry_policy.cpp b/ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.cpp
index 9e6b4e5a073..498833ec3f8 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_retry_policy.cpp
+++ b/ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.cpp
@@ -1,12 +1,12 @@
-#include "yql_s3_retry_policy.h"
+#include "yql_http_default_retry_policy.h"
 
 namespace NYql {
 
-IRetryPolicy<long>::TPtr GetS3RetryPolicy() {
+IRetryPolicy<long>::TPtr GetHTTPDefaultRetryPolicy() {
     return IRetryPolicy<long>::GetExponentialBackoffPolicy([](long httpCode) {
         switch (httpCode) {
             case 0: return ERetryErrorClass::ShortRetry;
-            case 503: return ERetryErrorClass::LongRetry;
+            case 503:return ERetryErrorClass::LongRetry;
             default: return ERetryErrorClass::NoRetry;
         }
     });
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_retry_policy.h b/ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.h
index 729985331fa..14057228daf 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_retry_policy.h
+++ b/ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.h
@@ -4,6 +4,6 @@
 
 namespace NYql {
 
-IRetryPolicy<long>::TPtr GetS3RetryPolicy();
+IRetryPolicy<long>::TPtr GetHTTPDefaultRetryPolicy();
 
 }
diff --git a/ydb/library/yql/providers/s3/actors/CMakeLists.txt b/ydb/library/yql/providers/s3/actors/CMakeLists.txt
index 910d8cab69a..fdeab19adc1 100644
--- a/ydb/library/yql/providers/s3/actors/CMakeLists.txt
+++ b/ydb/library/yql/providers/s3/actors/CMakeLists.txt
@@ -38,7 +38,6 @@ target_link_libraries(providers-s3-actors PUBLIC
 )
 target_sources(providers-s3-actors PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
-  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/actors/yql_s3_retry_policy.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/actors/yql_s3_sink_factory.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/actors/yql_s3_source_factory.cpp
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
index ed6c025b397..92e22875043 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
@@ -21,7 +21,6 @@
 #endif
 
 #include "yql_s3_read_actor.h"
-#include "yql_s3_retry_policy.h"
 
 #include <ydb/library/yql/minikql/mkql_string_util.h>
 #include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
@@ -31,6 +30,7 @@
 #include <ydb/library/yql/minikql/mkql_node_cast.h>
 #include <ydb/library/yql/minikql/mkql_terminator.h>
 #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+#include <ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.h>
 #include <ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h>
 #include <ydb/library/yql/utils/yql_panic.h>
 
@@ -150,7 +150,7 @@ public:
             const TPath& path = Paths[pathInd];
             Gateway->Download(Url + std::get<TString>(path),
                 Headers, std::min(std::get<size_t>(path), SizeLimit),
-                std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd + StartPathIndex), {}, GetS3RetryPolicy());
+                std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), std::placeholders::_1, pathInd + StartPathIndex), {}, GetHTTPDefaultRetryPolicy());
         };
     }
 
@@ -272,7 +272,7 @@ struct TRetryStuff {
         TString url,
         const IHTTPGateway::THeaders& headers,
         std::size_t sizeLimit
-    ) : Gateway(std::move(gateway)), Url(std::move(url)), Headers(headers), SizeLimit(sizeLimit), Offset(0U), RetryState(GetS3RetryPolicy()->CreateRetryState())
+    ) : Gateway(std::move(gateway)), Url(std::move(url)), Headers(headers), SizeLimit(sizeLimit), Offset(0U), RetryState(GetHTTPDefaultRetryPolicy()->CreateRetryState())
     {}
 
     const IHTTPGateway::TPtr Gateway;
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp
index 9eebb7a4126..adff44c2ea2 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp
@@ -1,8 +1,8 @@
 #include "yql_s3_write_actor.h"
-#include "yql_s3_retry_policy.h"
 
 #include <ydb/core/protos/services.pb.h>
 
+#include <ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.h>
 #include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
 #include <ydb/library/yql/providers/s3/compressors/factory.h>
 #include <ydb/library/yql/utils/yql_panic.h>
@@ -110,10 +110,10 @@ public:
             const auto size = Parts->Volume();
             InFlight += size;
             SentSize += size;
-            Gateway->Upload(Url, MakeHeader(), Parts->Pop(), std::bind(&TS3FileWriteActor::OnUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, std::placeholders::_1), true, GetS3RetryPolicy());
+            Gateway->Upload(Url, MakeHeader(), Parts->Pop(), std::bind(&TS3FileWriteActor::OnUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, std::placeholders::_1), true, GetHTTPDefaultRetryPolicy());
         } else {
             Become(&TS3FileWriteActor::InitialStateFunc);
-            Gateway->Upload(Url + "?uploads", MakeHeader(), 0, std::bind(&TS3FileWriteActor::OnUploadsCreated, ActorSystem, SelfId(), ParentId, std::placeholders::_1), false, GetS3RetryPolicy());
+            Gateway->Upload(Url + "?uploads", MakeHeader(), 0, std::bind(&TS3FileWriteActor::OnUploadsCreated, ActorSystem, SelfId(), ParentId, std::placeholders::_1), false, GetHTTPDefaultRetryPolicy());
         }
     }
 
@@ -195,7 +195,7 @@ private:
     static void OnPartUploadFinish(TActorSystem* actorSystem, TActorId selfId, TActorId parentId, size_t size, size_t index, IHTTPGateway::TResult&& response) {
         switch (response.index()) {
         case 0U: {
-            const auto& str = std::get<IHTTPGateway::TContent>(std::move(response)).Headers;
+            const auto& str = std::get<IHTTPGateway::TContent>(response).Headers;
 
             if (const auto p = str.find("etag: \""); p != TString::npos) {
                 if (const auto p1 = p + 7, p2 = str.find("\"", p1); p2 != TString::npos) {
@@ -280,7 +280,7 @@ private:
             Tags.emplace_back();
             InFlight += size;
             SentSize += size;
-            Gateway->Upload(Url + "?partNumber=" + std::to_string(index + 1) + "&uploadId=" + UploadId, MakeHeader(), std::move(part), std::bind(&TS3FileWriteActor::OnPartUploadFinish, ActorSystem, SelfId(), ParentId, size, index, std::placeholders::_1), true, GetS3RetryPolicy());
+            Gateway->Upload(Url + "?partNumber=" + std::to_string(index + 1) + "&uploadId=" + UploadId, MakeHeader(), std::move(part), std::bind(&TS3FileWriteActor::OnPartUploadFinish, ActorSystem, SelfId(), ParentId, size, index, std::placeholders::_1), true, GetHTTPDefaultRetryPolicy());
         }
     }
 
@@ -293,7 +293,7 @@ private:
         for (const auto& tag : Tags)
             xml << "<Part><PartNumber>" << ++i << "</PartNumber><ETag>" << tag << "</ETag></Part>" << Endl;
         xml << "</CompleteMultipartUpload>" << Endl;
-        Gateway->Upload(Url + "?uploadId=" + UploadId, MakeHeader(), xml, std::bind(&TS3FileWriteActor::OnMultipartUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, std::placeholders::_1), false, GetS3RetryPolicy());
+        Gateway->Upload(Url + "?uploadId=" + UploadId, MakeHeader(), xml, std::bind(&TS3FileWriteActor::OnMultipartUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, std::placeholders::_1), false, GetHTTPDefaultRetryPolicy());
     }
 
     IHTTPGateway::THeaders MakeHeader() const {
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_list.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_list.cpp
index 87cb666a5b2..8d4b72500b1 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_list.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_list.cpp
@@ -1,6 +1,7 @@
 #include "yql_s3_list.h"
 #include "yql_s3_path.h"
 
+#include <ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.h>
 #include <ydb/library/yql/utils/log/log.h>
 #include <ydb/library/yql/utils/url_builder.h>
 #include <ydb/library/yql/utils/yql_panic.h>
@@ -11,19 +12,12 @@
 #undef THROW
 #endif
 #include <library/cpp/xml/document/xml-document.h>
-#include <library/cpp/retry/retry_policy.h>
-
 #include <util/string/builder.h>
 
-
 namespace NYql {
 
 namespace {
 
-ERetryErrorClass RetryS3SlowDown(long httpResponseCode) {
-    return httpResponseCode == 503 ? ERetryErrorClass::LongRetry : ERetryErrorClass::NoRetry; // S3 Slow Down == 503
-}
-
 TString RegexFromWildcards(const std::string_view& pattern) {
     const auto& escaped = NS3::EscapeRegex(ToString(pattern));
     TStringBuilder result;
@@ -235,7 +229,7 @@ private:
         TString prefix;
         TResultFilter filter = MakeFilter(pattern, pathPrefix, prefix);
 
-        const auto retryPolicy = IRetryPolicy<long>::GetExponentialBackoffPolicy(RetryS3SlowDown);
+        const auto retryPolicy = GetHTTPDefaultRetryPolicy();
         TUrlBuilder urlBuilder(urlStr);
         const auto url = urlBuilder
             .AddUrlParam("list-type", "2")