diff options
author | hor911 <hor911@ydb.tech> | 2023-03-16 04:24:43 +0300 |
---|---|---|
committer | hor911 <hor911@ydb.tech> | 2023-03-16 04:24:43 +0300 |
commit | 7aa77e2e328f20250b7f1e1977cfbb29f71343db (patch) | |
tree | bce6b73d5d938e81460dd5e3de05daddd76b6684 | |
parent | af0ccdb33dabd69888206ac64acf95b632ac1b21 (diff) | |
download | ydb-7aa77e2e328f20250b7f1e1977cfbb29f71343db.tar.gz |
MakeHeaders dedup
5 files changed, 55 insertions, 41 deletions
diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp index 0ba8ddee48a..bb651a292b7 100644 --- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp +++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp @@ -937,4 +937,24 @@ IHTTPGateway::Make(const THttpGatewayConfig* httpGatewaysCfg, ::NMonitoring::TDy return gateway; } +IHTTPGateway::THeaders IHTTPGateway::MakeYcHeaders(const TString& requestId, const TString& token, const TString& contentType) { + + IHTTPGateway::THeaders result; + + if (requestId.empty()) { + throw yexception() << "RequestId is mandatory"; + } + result.push_back(TString("X-Request-ID:") + requestId); + + if (!token.empty()) { + result.push_back(TString("X-YaCloud-SubjectToken:") + token); + } + + if (!contentType.empty()) { + result.push_back(TString("Content-Type:") + contentType); + } + + return result; +} + } diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h index db58a4bae16..0bc2bf8c5e7 100644 --- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h +++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h @@ -113,6 +113,8 @@ public: const ::NMonitoring::TDynamicCounters::TCounterPtr& inflightCounter) = 0; virtual ui64 GetBuffersSizePerStream() = 0; + + static THeaders MakeYcHeaders(const TString& requestId, const TString& token = "", const TString& contentType = ""); }; } diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp index 819fdd1c421..5c8e35e11e3 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp @@ -771,8 +771,13 @@ public: auto id = index + StartPathIndex; const TString requestId = CreateGuidAsString(); LOG_D("TS3ReadActor", "Download: " << url << ", ID: " << id << ", request id: [" << requestId << "]"); - Gateway->Download(url, MakeHeaders(Token, requestId), 0U, std::min(size, SizeLimit), - std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), requestId, std::placeholders::_1, id, path), {}, RetryPolicy); + Gateway->Download(url, + IHTTPGateway::MakeYcHeaders(requestId, Token), + 0U, + std::min(size, SizeLimit), + std::bind(&TS3ReadActor::OnDownloadFinished, ActorSystem, SelfId(), requestId, std::placeholders::_1, id, path), + {}, + RetryPolicy); } TObjectPath ReadPathFromCache() { @@ -955,14 +960,6 @@ private: TActorBootstrapped<TS3ReadActor>::PassAway(); } - static IHTTPGateway::THeaders MakeHeaders(const TString& token, const TString& requestId) { - IHTTPGateway::THeaders headers{TString{"X-Request-ID:"} += requestId}; - if (token) { - headers.emplace_back(TString("X-YaCloud-SubjectToken:") += token); - } - return headers; - } - private: const TS3ReadActorFactoryConfig ReadActorFactoryCfg; const IHTTPGateway::TPtr Gateway; @@ -2208,7 +2205,7 @@ public: auto stuff = std::make_shared<TRetryStuff>( Gateway, Url + objectPath.Path, - MakeHeaders(Token, requestId), + IHTTPGateway::MakeYcHeaders(requestId, Token), objectPath.Size, TxId, requestId, @@ -2417,14 +2414,6 @@ private: TActorBootstrapped<TS3StreamReadActor>::PassAway(); } - static IHTTPGateway::THeaders MakeHeaders(const TString& token, const TString& requestId) { - IHTTPGateway::THeaders headers{TString{"X-Request-ID:"} += requestId}; - if (token) { - headers.emplace_back(TString("X-YaCloud-SubjectToken:") += token); - } - return headers; - } - void MaybePause() { if (!Paused && QueueTotalDataSize >= ReadActorFactoryCfg.DataInflight) { for (const auto actorId : CoroActors) { diff --git a/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp b/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp index ec5740cda40..bc0f54e7f25 100644 --- a/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp +++ b/ydb/library/yql/providers/s3/actors/yql_s3_write_actor.cpp @@ -150,10 +150,20 @@ public: const size_t size = Max<size_t>(Parts->Volume(), 1); InFlight += size; SentSize += size; - Gateway->Upload(Url, MakeHeaders(RequestId), Parts->Pop(), std::bind(&TS3FileWriteActor::OnUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, size, std::placeholders::_1), true, RetryPolicy); + Gateway->Upload(Url, + IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo()), + Parts->Pop(), + std::bind(&TS3FileWriteActor::OnUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, size, std::placeholders::_1), + true, + RetryPolicy); } else { Become(&TS3FileWriteActor::MultipartInitialStateFunc); - Gateway->Upload(Url + "?uploads", MakeHeaders(RequestId), 0, std::bind(&TS3FileWriteActor::OnUploadsCreated, ActorSystem, SelfId(), ParentId, RequestId, std::placeholders::_1), false, RetryPolicy); + Gateway->Upload(Url + "?uploads", + IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo()), + 0, + std::bind(&TS3FileWriteActor::OnUploadsCreated, ActorSystem, SelfId(), ParentId, RequestId, std::placeholders::_1), + false, + RetryPolicy); } } @@ -354,7 +364,12 @@ private: Tags.emplace_back(); InFlight += size; SentSize += size; - Gateway->Upload(Url + "?partNumber=" + std::to_string(index + 1) + "&uploadId=" + UploadId, MakeHeaders(RequestId), std::move(part), std::bind(&TS3FileWriteActor::OnPartUploadFinish, ActorSystem, SelfId(), ParentId, size, index, RequestId, std::placeholders::_1), true, RetryPolicy); + Gateway->Upload(Url + "?partNumber=" + std::to_string(index + 1) + "&uploadId=" + UploadId, + IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo()), + std::move(part), + std::bind(&TS3FileWriteActor::OnPartUploadFinish, ActorSystem, SelfId(), ParentId, size, index, RequestId, std::placeholders::_1), + true, + RetryPolicy); } } @@ -367,18 +382,12 @@ private: for (const auto& tag : Tags) xml << "<Part><PartNumber>" << ++i << "</PartNumber><ETag>" << tag << "</ETag></Part>" << Endl; xml << "</CompleteMultipartUpload>" << Endl; - Gateway->Upload(Url + "?uploadId=" + UploadId, MakeHeaders(RequestId, "application/xml"sv), xml, std::bind(&TS3FileWriteActor::OnMultipartUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, SentSize, std::placeholders::_1), false, RetryPolicy); - } - - IHTTPGateway::THeaders MakeHeaders(const TString& requestId, std::string_view contentType = "") const { - auto headers = IHTTPGateway::THeaders{ TString{"X-Request-ID:"} += requestId }; - if (const auto& token = CredProvider->GetAuthInfo(); !token.empty()) { - headers.push_back(TString("X-YaCloud-SubjectToken:") += token); - } - if (!contentType.empty()) { - headers.push_back(TString("Content-Type:") += contentType); - } - return headers; + Gateway->Upload(Url + "?uploadId=" + UploadId, + IHTTPGateway::MakeYcHeaders(RequestId, CredProvider->GetAuthInfo(), "application/xml"), + xml, + std::bind(&TS3FileWriteActor::OnMultipartUploadFinish, ActorSystem, SelfId(), ParentId, Key, Url, RequestId, SentSize, std::placeholders::_1), + false, + RetryPolicy); } size_t InFlight = 0ULL; diff --git a/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp b/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp index f2437741f52..917e55f96c4 100644 --- a/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp +++ b/ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp @@ -273,12 +273,6 @@ public: private: static void SubmitRequestIntoGateway(TListingContext& ctx) { - IHTTPGateway::THeaders headers; - if (!ctx.ListingRequest.Token.empty()) { - headers.emplace_back("X-YaCloud-SubjectToken:" + ctx.ListingRequest.Token); - } - headers.emplace_back(TString{"X-Request-ID:"} + ctx.RequestId); - TUrlBuilder urlBuilder(ctx.ListingRequest.Url); urlBuilder.AddUrlParam("list-type", "2") .AddUrlParam("prefix", ctx.ListingRequest.Prefix) @@ -299,7 +293,7 @@ private: auto retryPolicy = ctx.RetryPolicy; gateway->Download( urlBuilder.Build(), - headers, + IHTTPGateway::MakeYcHeaders(ctx.RequestId, ctx.ListingRequest.Token), 0U, 0U, CallbackFactoryMethod(std::move(ctx)), |