aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhor911 <hor911@ydb.tech>2023-03-16 04:24:43 +0300
committerhor911 <hor911@ydb.tech>2023-03-16 04:24:43 +0300
commit7aa77e2e328f20250b7f1e1977cfbb29f71343db (patch)
treebce6b73d5d938e81460dd5e3de05daddd76b6684
parentaf0ccdb33dabd69888206ac64acf95b632ac1b21 (diff)
downloadydb-7aa77e2e328f20250b7f1e1977cfbb29f71343db.tar.gz
MakeHeaders dedup
-rw-r--r--ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp20
-rw-r--r--ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h2
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp27
-rw-r--r--ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp39
-rw-r--r--ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp8
5 files changed, 55 insertions, 41 deletions
diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp
index 0ba8ddee48a..bb651a292b7 100644
--- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp
+++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp
@@ -937,4 +937,24 @@ IHTTPGateway::Make(const THttpGatewayConfig* httpGatewaysCfg, ::NMonitoring::TDy
return gateway;
}
+IHTTPGateway::THeaders IHTTPGateway::MakeYcHeaders(const TString& requestId, const TString& token, const TString& contentType) {
+
+ IHTTPGateway::THeaders result;
+
+ if (requestId.empty()) {
+ throw yexception() << "RequestId is mandatory";
+ }
+ result.push_back(TString("X-Request-ID:") + requestId);
+
+ if (!token.empty()) {
+ result.push_back(TString("X-YaCloud-SubjectToken:") + token);
+ }
+
+ if (!contentType.empty()) {
+ result.push_back(TString("Content-Type:") + contentType);
+ }
+
+ return result;
+}
+
}
diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h
index db58a4bae16..0bc2bf8c5e7 100644
--- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h
+++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h
@@ -113,6 +113,8 @@ public:
const ::NMonitoring::TDynamicCounters::TCounterPtr& inflightCounter) = 0;
virtual ui64 GetBuffersSizePerStream() = 0;
+
+ static THeaders MakeYcHeaders(const TString& requestId, const TString& token = "", const TString& contentType = "");
};
}
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
index 819fdd1c421..5c8e35e11e3 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp
@@ -771,8 +771,13 @@ public:
auto id = index + StartPathIndex;
const TString requestId = CreateGuidAsString();
LOG_D("TS3ReadActor", "Download: " << url << ", ID: " << id << ", request id: [" << requestId << "]");
- Gateway->Download(url, MakeHeaders(Token, requestId), 0U, std::min(size, SizeLimit),
- std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), requestId, std::placeholders::_1, id, path), {}, RetryPolicy);
+ Gateway->Download(url,
+ IHTTPGateway::MakeYcHeaders(requestId, Token),
+ 0U,
+ std::min(size, SizeLimit),
+ std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), requestId, std::placeholders::_1, id, path),
+ {},
+ RetryPolicy);
}
TObjectPath ReadPathFromCache() {
@@ -955,14 +960,6 @@ private:
TActorBootstrapped<TS3ReadActor>::PassAway();
}
- static IHTTPGateway::THeaders MakeHeaders(const TString& token, const TString& requestId) {
- IHTTPGateway::THeaders headers{TString{"X-Request-ID:"} += requestId};
- if (token) {
- headers.emplace_back(TString("X-YaCloud-SubjectToken:") += token);
- }
- return headers;
- }
-
private:
const TS3ReadActorFactoryConfig ReadActorFactoryCfg;
const IHTTPGateway::TPtr Gateway;
@@ -2208,7 +2205,7 @@ public:
auto stuff = std::make_shared<TRetryStuff>(
Gateway,
Url + objectPath.Path,
- MakeHeaders(Token, requestId),
+ IHTTPGateway::MakeYcHeaders(requestId, Token),
objectPath.Size,
TxId,
requestId,
@@ -2417,14 +2414,6 @@ private:
TActorBootstrapped<TS3StreamReadActor>::PassAway();
}
- static IHTTPGateway::THeaders MakeHeaders(const TString& token, const TString& requestId) {
- IHTTPGateway::THeaders headers{TString{"X-Request-ID:"} += requestId};
- if (token) {
- headers.emplace_back(TString("X-YaCloud-SubjectToken:") += token);
- }
- return headers;
- }
-
void MaybePause() {
if (!Paused && QueueTotalDataSize >= ReadActorFactoryCfg.DataInflight) {
for (const auto actorId : CoroActors) {
diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp
index ec5740cda40..bc0f54e7f25 100644
--- a/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp
+++ b/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp
@@ -150,10 +150,20 @@ public:
const size_t size = Max<size_t>(Parts->Volume(), 1);
InFlight += size;
SentSize += size;
- Gateway->Upload(Url, MakeHeaders(RequestId), Parts->Pop(), std::bind(&TS3FileWriteActor::OnUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, size, std::placeholders::_1), true, RetryPolicy);
+ Gateway->Upload(Url,
+ IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo()),
+ Parts->Pop(),
+ std::bind(&TS3FileWriteActor::OnUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, size, std::placeholders::_1),
+ true,
+ RetryPolicy);
} else {
Become(&TS3FileWriteActor::MultipartInitialStateFunc);
- Gateway->Upload(Url + "?uploads", MakeHeaders(RequestId), 0, std::bind(&TS3FileWriteActor::OnUploadsCreated, ActorSystem, SelfId(), ParentId, RequestId, std::placeholders::_1), false, RetryPolicy);
+ Gateway->Upload(Url + "?uploads",
+ IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo()),
+ 0,
+ std::bind(&TS3FileWriteActor::OnUploadsCreated, ActorSystem, SelfId(), ParentId, RequestId, std::placeholders::_1),
+ false,
+ RetryPolicy);
}
}
@@ -354,7 +364,12 @@ private:
Tags.emplace_back();
InFlight += size;
SentSize += size;
- Gateway->Upload(Url + "?partNumber=" + std::to_string(index + 1) + "&uploadId=" + UploadId, MakeHeaders(RequestId), std::move(part), std::bind(&TS3FileWriteActor::OnPartUploadFinish, ActorSystem, SelfId(), ParentId, size, index, RequestId, std::placeholders::_1), true, RetryPolicy);
+ Gateway->Upload(Url + "?partNumber=" + std::to_string(index + 1) + "&uploadId=" + UploadId,
+ IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo()),
+ std::move(part),
+ std::bind(&TS3FileWriteActor::OnPartUploadFinish, ActorSystem, SelfId(), ParentId, size, index, RequestId, std::placeholders::_1),
+ true,
+ RetryPolicy);
}
}
@@ -367,18 +382,12 @@ private:
for (const auto& tag : Tags)
xml << "<Part><PartNumber>" << ++i << "</PartNumber><ETag>" << tag << "</ETag></Part>" << Endl;
xml << "</CompleteMultipartUpload>" << Endl;
- Gateway->Upload(Url + "?uploadId=" + UploadId, MakeHeaders(RequestId, "application/xml"sv), xml, std::bind(&TS3FileWriteActor::OnMultipartUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, SentSize, std::placeholders::_1), false, RetryPolicy);
- }
-
- IHTTPGateway::THeaders MakeHeaders(const TString& requestId, std::string_view contentType = "") const {
- auto headers = IHTTPGateway::THeaders{ TString{"X-Request-ID:"} += requestId };
- if (const auto& token = CredProvider->GetAuthInfo(); !token.empty()) {
- headers.push_back(TString("X-YaCloud-SubjectToken:") += token);
- }
- if (!contentType.empty()) {
- headers.push_back(TString("Content-Type:") += contentType);
- }
- return headers;
+ Gateway->Upload(Url + "?uploadId=" + UploadId,
+ IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo(), "application/xml"),
+ xml,
+ std::bind(&TS3FileWriteActor::OnMultipartUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, SentSize, std::placeholders::_1),
+ false,
+ RetryPolicy);
}
size_t InFlight = 0ULL;
diff --git a/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp b/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp
index f2437741f52..917e55f96c4 100644
--- a/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp
+++ b/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp
@@ -273,12 +273,6 @@ public:
private:
static void SubmitRequestIntoGateway(TListingContext& ctx) {
- IHTTPGateway::THeaders headers;
- if (!ctx.ListingRequest.Token.empty()) {
- headers.emplace_back("X-YaCloud-SubjectToken:" + ctx.ListingRequest.Token);
- }
- headers.emplace_back(TString{"X-Request-ID:"} + ctx.RequestId);
-
TUrlBuilder urlBuilder(ctx.ListingRequest.Url);
urlBuilder.AddUrlParam("list-type", "2")
.AddUrlParam("prefix", ctx.ListingRequest.Prefix)
@@ -299,7 +293,7 @@ private:
auto retryPolicy = ctx.RetryPolicy;
gateway->Download(
urlBuilder.Build(),
- headers,
+ IHTTPGateway::MakeYcHeaders(ctx.RequestId, ctx.ListingRequest.Token),
0U,
0U,
CallbackFactoryMethod(std::move(ctx)),