diff options
author | dakovalkov <dakovalkov@yandex-team.com> | 2023-12-03 13:33:55 +0300 |
---|---|---|
committer | dakovalkov <dakovalkov@yandex-team.com> | 2023-12-03 14:04:39 +0300 |
commit | 2a718325637e5302334b6d0a6430f63168f8dbb3 (patch) | |
tree | 64be81080b7df9ec1d86d053a0c394ae53fcf1fe /contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http | |
parent | e0d94a470142d95c3007e9c5d80380994940664a (diff) | |
download | ydb-2a718325637e5302334b6d0a6430f63168f8dbb3.tar.gz |
Update contrib/libs/aws-sdk-cpp to 1.11.37
Diffstat (limited to 'contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http')
8 files changed, 359 insertions, 87 deletions
diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpClientFactory.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpClientFactory.cpp index a556e39a5d..a08b21f9b5 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpClientFactory.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpClientFactory.cpp @@ -121,9 +121,11 @@ namespace Aws void InitStaticState() override { + AWS_LOGSTREAM_DEBUG(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, "Initializing Http Static State"); #if ENABLE_CURL_CLIENT if(s_InitCleanupCurlFlag) { + AWS_LOGSTREAM_DEBUG(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, "Initializing Curl Http Client"); CurlHttpClient::InitGlobalState(); } #if !defined (_WIN32) @@ -139,9 +141,11 @@ namespace Aws virtual void CleanupStaticState() override { + AWS_LOGSTREAM_DEBUG(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, "Cleanup Http Static State"); #if ENABLE_CURL_CLIENT if(s_InitCleanupCurlFlag) { + AWS_LOGSTREAM_DEBUG(HTTP_CLIENT_FACTORY_ALLOCATION_TAG, "Cleanup Curl Http Client"); CurlHttpClient::CleanupGlobalState(); } #endif diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpRequest.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpRequest.cpp index 95cb626c22..1f109c86a9 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpRequest.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpRequest.cpp @@ -4,37 +4,88 @@ */ #include <aws/core/http/HttpRequest.h> - +#include <aws/core/utils/memory/stl/AWSStringStream.h> +#include <aws/http/request_response.h> +#include <aws/crt/Types.h> +#include <aws/crt/http/HttpRequestResponse.h> namespace Aws { -namespace Http -{ + namespace Http + { -const char DATE_HEADER[] = "date"; -const char AWS_DATE_HEADER[] = "X-Amz-Date"; -const char AWS_SECURITY_TOKEN[] = "X-Amz-Security-Token"; -const char ACCEPT_HEADER[] = "accept"; -const char ACCEPT_CHAR_SET_HEADER[] = "accept-charset"; -const char ACCEPT_ENCODING_HEADER[] = "accept-encoding"; -const char AUTHORIZATION_HEADER[] = "authorization"; -const char AWS_AUTHORIZATION_HEADER[] = "authorization"; -const char COOKIE_HEADER[] = "cookie"; -const char CONTENT_LENGTH_HEADER[] = "content-length"; -const char CONTENT_TYPE_HEADER[] = "content-type"; -const char TRANSFER_ENCODING_HEADER[] = "transfer-encoding"; -const char USER_AGENT_HEADER[] = "user-agent"; -const char VIA_HEADER[] = "via"; -const char HOST_HEADER[] = "host"; -const char AMZ_TARGET_HEADER[] = "x-amz-target"; -const char X_AMZ_EXPIRES_HEADER[] = "X-Amz-Expires"; -const char CONTENT_MD5_HEADER[] = "content-md5"; -const char API_VERSION_HEADER[] = "x-amz-api-version"; -const char SDK_INVOCATION_ID_HEADER[] = "amz-sdk-invocation-id"; -const char SDK_REQUEST_HEADER[] = "amz-sdk-request"; -const char CHUNKED_VALUE[] = "chunked"; + const char DATE_HEADER[] = "date"; + const char AWS_DATE_HEADER[] = "X-Amz-Date"; + const char AWS_SECURITY_TOKEN[] = "X-Amz-Security-Token"; + const char ACCEPT_HEADER[] = "accept"; + const char ACCEPT_CHAR_SET_HEADER[] = "accept-charset"; + const char ACCEPT_ENCODING_HEADER[] = "accept-encoding"; + const char AUTHORIZATION_HEADER[] = "authorization"; + const char AWS_AUTHORIZATION_HEADER[] = "authorization"; + const char COOKIE_HEADER[] = "cookie"; + const char DECODED_CONTENT_LENGTH_HEADER[] = "x-amz-decoded-content-length"; + const char CONTENT_LENGTH_HEADER[] = "content-length"; + const char CONTENT_TYPE_HEADER[] = "content-type"; + const char CONTENT_ENCODING_HEADER[] = "content-encoding"; + const char TRANSFER_ENCODING_HEADER[] = "transfer-encoding"; + const char USER_AGENT_HEADER[] = "user-agent"; + const char VIA_HEADER[] = "via"; + const char HOST_HEADER[] = "host"; + const char AMZ_TARGET_HEADER[] = "x-amz-target"; + const char X_AMZ_EXPIRES_HEADER[] = "X-Amz-Expires"; + const char CONTENT_MD5_HEADER[] = "content-md5"; + const char API_VERSION_HEADER[] = "x-amz-api-version"; + const char AWS_TRAILER_HEADER[] = "x-amz-trailer"; + const char SDK_INVOCATION_ID_HEADER[] = "amz-sdk-invocation-id"; + const char SDK_REQUEST_HEADER[] = "amz-sdk-request"; + const char CHUNKED_VALUE[] = "chunked"; + const char AWS_CHUNKED_VALUE[] = "aws-chunked"; + const char X_AMZN_TRACE_ID_HEADER[] = "X-Amzn-Trace-Id"; + const char ALLOCATION_TAG[] = "HttpRequestConversion"; + const char X_AMZN_ERROR_TYPE[] = "x-amzn-errortype"; -} // Http -} // Aws + std::shared_ptr<Aws::Crt::Http::HttpRequest> HttpRequest::ToCrtHttpRequest() + { + auto request = Aws::MakeShared<Aws::Crt::Http::HttpRequest>(ALLOCATION_TAG); + request->SetBody([&]() -> std::shared_ptr<IOStream> { + const std::shared_ptr<Aws::IOStream>& body = GetContentBody(); + if (body) { + return body; + } + // Return an empty string stream for no body + return Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, ""); + }()); + auto headers = GetHeaders(); + for (const auto& it: headers) + { + Aws::Crt::Http::HttpHeader header; + header.name = Aws::Crt::ByteCursorFromCString(it.first.c_str()); + header.value = Aws::Crt::ByteCursorFromCString(it.second.c_str()); + request->AddHeader(header); + } + // Need a different URL encoding here. + // CRT sigv4 don't any encoding if double encoding is off, need to encode the path before passing to CRT. + const URI& uri = m_uri; + Aws::StringStream ss; + Aws::StringStream port; + if (uri.GetScheme() == Scheme::HTTP && uri.GetPort() != HTTP_DEFAULT_PORT) + { + port << ":" << uri.GetPort(); + } + else if (uri.GetScheme() == Scheme::HTTPS && uri.GetPort() != HTTPS_DEFAULT_PORT) + { + port << ":" << uri.GetPort(); + } + ss << SchemeMapper::ToString(uri.GetScheme()) << SEPARATOR << uri.GetAuthority() << port.str() + << ((uri.GetPath() == "/") ? "" : URI::URLEncodePath(uri.GetPath())) + << uri.GetQueryString(); + request->SetPath(Aws::Crt::ByteCursorFromCString(ss.str().c_str())); + const char *method = HttpMethodMapper::GetNameForHttpMethod(m_method); + request->SetMethod(Aws::Crt::ByteCursorFromCString(method)); + return request; + } + + } // Http +} // Aws diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpResponse.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpResponse.cpp new file mode 100644 index 0000000000..d4e0833653 --- /dev/null +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/HttpResponse.cpp @@ -0,0 +1,24 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include <aws/core/http/HttpResponse.h> + +#include <aws/core/utils/memory/stl/AWSStreamFwd.h> +#include <aws/core/utils/StringUtils.h> + +namespace Aws +{ + namespace Http + { + /** + * Overload ostream operator<< for HttpResponseCode enum class for a prettier output such as "200" and not "<C8-00 00-00>" + */ + Aws::OStream& operator<< (Aws::OStream& oStream, HttpResponseCode code) + { + oStream << Aws::Utils::StringUtils::to_string(static_cast<typename std::underlying_type<HttpResponseCode>::type>(code)); + return oStream; + } + } // Http +} // Aws diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/URI.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/URI.cpp index a2239df54b..0bc3c09245 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/URI.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/URI.cpp @@ -5,9 +5,8 @@ #include <aws/core/http/URI.h> -#include <aws/core/utils/StringUtils.h> -#include <aws/core/utils/memory/stl/AWSStringStream.h> #include <aws/core/utils/memory/stl/AWSSet.h> +#include <aws/core/utils/logging/LogMacros.h> #include <cstdlib> #include <cctype> @@ -25,10 +24,52 @@ namespace Http const char* SEPARATOR = "://"; +bool s_compliantRfc3986Encoding = false; +void SetCompliantRfc3986Encoding(bool compliant) { s_compliantRfc3986Encoding = compliant; } + +Aws::String urlEncodeSegment(const Aws::String& segment) +{ + // consolidates legacy escaping logic into one local method + if (s_compliantRfc3986Encoding) + { + return StringUtils::URLEncode(segment.c_str()); + } + else + { + Aws::StringStream ss; + ss << std::hex << std::uppercase; + for(unsigned char c : segment) // alnum results in UB if the value of c is not unsigned char & is not EOF + { + // RFC 3986 §2.3 unreserved characters + if (StringUtils::IsAlnum(c)) + { + ss << c; + continue; + } + switch(c) + { + // §2.3 unreserved characters + // The path section of the URL allows unreserved characters to appear unescaped + case '-': case '_': case '.': case '~': + // RFC 3986 §2.2 Reserved characters + // NOTE: this implementation does not accurately implement the RFC on purpose to accommodate for + // discrepancies in the implementations of URL encoding between AWS services for legacy reasons. + case '$': case '&': case ',': + case ':': case '=': case '@': + ss << c; + break; + default: + ss << '%' << std::setfill('0') << std::setw(2) << (int)c << std::setw(0); + } + } + return ss.str(); + } +} + } // namespace Http } // namespace Aws -URI::URI() : m_scheme(Scheme::HTTP), m_port(HTTP_DEFAULT_PORT) +URI::URI() : m_scheme(Scheme::HTTP), m_port(HTTP_DEFAULT_PORT), m_pathHasTrailingSlash(false) { } @@ -102,7 +143,7 @@ void URI::SetScheme(Scheme value) Aws::String URI::URLEncodePathRFC3986(const Aws::String& path) { - if(path.empty()) + if (path.empty()) { return path; } @@ -114,34 +155,10 @@ Aws::String URI::URLEncodePathRFC3986(const Aws::String& path) // escape characters appearing in a URL path according to RFC 3986 for (const auto& segment : pathParts) { - ss << '/'; - for(unsigned char c : segment) // alnum results in UB if the value of c is not unsigned char & is not EOF - { - // §2.3 unreserved characters - if (StringUtils::IsAlnum(c)) - { - ss << c; - continue; - } - switch(c) - { - // §2.3 unreserved characters - case '-': case '_': case '.': case '~': - // The path section of the URL allow reserved characters to appear unescaped - // RFC 3986 §2.2 Reserved characters - // NOTE: this implementation does not accurately implement the RFC on purpose to accommodate for - // discrepancies in the implementations of URL encoding between AWS services for legacy reasons. - case '$': case '&': case ',': - case ':': case '=': case '@': - ss << c; - break; - default: - ss << '%' << std::setfill('0') << std::setw(2) << (int)((unsigned char)c) << std::setw(0); - } - } + ss << '/' << urlEncodeSegment(segment); } - //if the last character was also a slash, then add that back here. + // if the last character was also a slash, then add that back here. if (path.back() == '/') { ss << '/'; @@ -176,23 +193,65 @@ Aws::String URI::URLEncodePath(const Aws::String& path) } } -void URI::SetPath(const Aws::String& value) +Aws::String URI::GetPath() const { - const Aws::Vector<Aws::String> pathParts = StringUtils::Split(value, '/'); - Aws::String path; - path.reserve(value.length() + 1/* in case we have to append slash before the path. */); + Aws::String path = ""; - for (const auto& segment : pathParts) + for (auto const& segment : m_pathSegments) { path.push_back('/'); path.append(segment); } - if (value.back() == '/') + if (m_pathSegments.empty() || m_pathHasTrailingSlash) { path.push_back('/'); } - m_path = std::move(path); + + return path; +} + +Aws::String URI::GetURLEncodedPath() const +{ + Aws::StringStream ss; + + for (auto const& segment : m_pathSegments) + { + ss << '/' << StringUtils::URLEncode(segment.c_str()); + } + + if (m_pathSegments.empty() || m_pathHasTrailingSlash) + { + ss << '/'; + } + + return ss.str(); +} + +Aws::String URI::GetURLEncodedPathRFC3986() const +{ + Aws::StringStream ss; + ss << std::hex << std::uppercase; + + // escape characters appearing in a URL path according to RFC 3986 + // (mostly; there is some non-standards legacy support that can be disabled) + for (const auto& segment : m_pathSegments) + { + ss << '/' << urlEncodeSegment(segment); + } + + if (m_pathSegments.empty() || m_pathHasTrailingSlash) + { + ss << '/'; + } + + return ss.str(); +} + +void URI::SetPath(const Aws::String& value) +{ + m_pathSegments.clear(); + AddPathSegments(value); } //ugh, this isn't even part of the canonicalization spec. It is part of how our services have implemented their signers though.... @@ -347,9 +406,9 @@ Aws::String URI::GetURIString(bool includeQueryString) const ss << ":" << m_port; } - if(m_path != "/") + if (!m_pathSegments.empty()) { - ss << URLEncodePathRFC3986(m_path); + ss << GetURLEncodedPathRFC3986(); } if(includeQueryString) @@ -397,10 +456,26 @@ void URI::ExtractAndSetAuthority(const Aws::String& uri) authorityStart += 3; } - size_t posOfEndOfAuthorityPort = uri.find(':', authorityStart); - size_t posOfEndOfAuthoritySlash = uri.find('/', authorityStart); - size_t posOfEndOfAuthorityQuery = uri.find('?', authorityStart); - size_t posEndOfAuthority = (std::min)({posOfEndOfAuthorityPort, posOfEndOfAuthoritySlash, posOfEndOfAuthorityQuery}); + size_t posEndOfAuthority=0; + // are we extracting an ipv6 address? + if (uri.length() > authorityStart && uri.at(authorityStart) == '[') + { + posEndOfAuthority = uri.find(']', authorityStart); + if (posEndOfAuthority == Aws::String::npos) { + AWS_LOGSTREAM_ERROR("Uri", "Malformed uri: " << uri.c_str()); + } + else + { + ++posEndOfAuthority; + } + } + else + { + size_t posOfEndOfAuthorityPort = uri.find(':', authorityStart); + size_t posOfEndOfAuthoritySlash = uri.find('/', authorityStart); + size_t posOfEndOfAuthorityQuery = uri.find('?', authorityStart); + posEndOfAuthority = (std::min)({posOfEndOfAuthorityPort, posOfEndOfAuthoritySlash, posOfEndOfAuthorityQuery}); + } if (posEndOfAuthority == Aws::String::npos) { posEndOfAuthority = uri.length(); @@ -422,11 +497,25 @@ void URI::ExtractAndSetPort(const Aws::String& uri) authorityStart += 3; } - size_t positionOfPortDelimiter = uri.find(':', authorityStart); + size_t portSearchStart = authorityStart; + // are we extracting an ipv6 address? + if (uri.length() > portSearchStart && uri.at(portSearchStart) == '[') + { + size_t posEndOfAuthority = uri.find(']', portSearchStart); + if (posEndOfAuthority == Aws::String::npos) { + AWS_LOGSTREAM_ERROR("Uri", "Malformed uri: " << uri.c_str()); + } + else + { + portSearchStart = posEndOfAuthority; + } + } + + size_t positionOfPortDelimiter = uri.find(':', portSearchStart); bool hasPort = positionOfPortDelimiter != Aws::String::npos; - if ((uri.find('/', authorityStart) < positionOfPortDelimiter) || (uri.find('?', authorityStart) < positionOfPortDelimiter)) + if ((uri.find('/', portSearchStart) < positionOfPortDelimiter) || (uri.find('?', portSearchStart) < positionOfPortDelimiter)) { hasPort = false; } @@ -506,5 +595,5 @@ Aws::String URI::GetFormParameters() const bool URI::CompareURIParts(const URI& other) const { - return m_scheme == other.m_scheme && m_authority == other.m_authority && m_path == other.m_path && m_queryString == other.m_queryString; + return m_scheme == other.m_scheme && m_authority == other.m_authority && GetPath() == other.GetPath() && m_queryString == other.m_queryString; } diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHandleContainer.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHandleContainer.cpp index 1a965cd795..a6684c640a 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHandleContainer.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHandleContainer.cpp @@ -43,7 +43,7 @@ CURL* CurlHandleContainer::AcquireCurlHandle() } CURL* handle = m_handleContainer.Acquire(); - AWS_LOGSTREAM_INFO(CURL_HANDLE_CONTAINER_TAG, "Connection has been released. Continuing."); + AWS_LOGSTREAM_DEBUG(CURL_HANDLE_CONTAINER_TAG, "Connection has been released. Continuing."); AWS_LOGSTREAM_DEBUG(CURL_HANDLE_CONTAINER_TAG, "Returning connection handle " << handle); return handle; } @@ -52,6 +52,9 @@ void CurlHandleContainer::ReleaseCurlHandle(CURL* handle) { if (handle) { +#if LIBCURL_VERSION_NUM >= 0x074D00 // 7.77.0 + curl_easy_setopt(handle, CURLOPT_COOKIEFILE, NULL); // workaround a mem leak on curl +#endif curl_easy_reset(handle); SetDefaultOptionsOnHandle(handle); AWS_LOGSTREAM_DEBUG(CURL_HANDLE_CONTAINER_TAG, "Releasing curl handle " << handle); diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp index 95132f5df0..0f64b15062 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/curl/CurlHttpClient.cpp @@ -7,9 +7,12 @@ #include <aws/core/http/HttpRequest.h> #include <aws/core/http/standard/StandardHttpResponse.h> #include <aws/core/utils/StringUtils.h> +#include <aws/core/utils/HashingUtils.h> #include <aws/core/utils/logging/LogMacros.h> #include <aws/core/utils/ratelimiter/RateLimiterInterface.h> #include <aws/core/utils/DateTime.h> +#include <aws/core/utils/crypto/Hash.h> +#include <aws/core/utils/Outcome.h> #include <aws/core/monitoring/HttpClientMetrics.h> #include <cassert> #include <algorithm> @@ -146,17 +149,34 @@ struct CurlReadCallbackContext m_client(client), m_curlHandle(curlHandle), m_rateLimiter(limiter), - m_request(request) + m_request(request), + m_chunkEnd(false) {} const CurlHttpClient* m_client; CURL* m_curlHandle; Aws::Utils::RateLimits::RateLimiterInterface* m_rateLimiter; HttpRequest* m_request; + bool m_chunkEnd; }; static const char* CURL_HTTP_CLIENT_TAG = "CurlHttpClient"; +static int64_t GetContentLengthFromHeader(CURL* connectionHandle, + bool& hasContentLength) { +#if LIBCURL_VERSION_NUM >= 0x073700 // 7.55.0 + curl_off_t contentLength = {}; + CURLcode res = curl_easy_getinfo( + connectionHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &contentLength); +#else + double contentLength = {}; + CURLcode res = curl_easy_getinfo( + connectionHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength); +#endif + hasContentLength = (res == CURLE_OK) && (contentLength != -1); + return hasContentLength ? static_cast<int64_t>(contentLength) : -1; +} + static size_t WriteData(char* ptr, size_t size, size_t nmemb, void* userdata) { if (ptr) @@ -176,8 +196,13 @@ static size_t WriteData(char* ptr, size_t size, size_t nmemb, void* userdata) context->m_rateLimiter->ApplyAndPayForCost(static_cast<int64_t>(sizeToWrite)); } + for (const auto& hashIterator : context->m_request->GetResponseValidationHashes()) + { + hashIterator.second->Update(reinterpret_cast<unsigned char*>(ptr), sizeToWrite); + } + response->GetResponseBody().write(ptr, static_cast<std::streamsize>(sizeToWrite)); - if (context->m_request->IsEventStreamRequest()) + if (context->m_request->IsEventStreamRequest() && !response->HasHeader(Aws::Http::X_AMZN_ERROR_TYPE)) { response->GetResponseBody().flush(); } @@ -214,8 +239,7 @@ static size_t WriteHeader(char* ptr, size_t size, size_t nmemb, void* userdata) return 0; } - -static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata) +static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata, bool isStreaming) { CurlReadCallbackContext* context = reinterpret_cast<CurlReadCallbackContext*>(userdata); if(context == nullptr) @@ -232,10 +256,20 @@ static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata) HttpRequest* request = context->m_request; const std::shared_ptr<Aws::IOStream>& ioStream = request->GetContentBody(); - const size_t amountToRead = size * nmemb; + size_t amountToRead = size * nmemb; + bool isAwsChunked = request->HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) && + request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE; + // aws-chunk = hex(chunk-size) + CRLF + chunk-data + CRLF + // Needs to reserve bytes of sizeof(hex(chunk-size)) + sizeof(CRLF) + sizeof(CRLF) + if (isAwsChunked) + { + Aws::String amountToReadHexString = Aws::Utils::StringUtils::ToHexString(amountToRead); + amountToRead -= (amountToReadHexString.size() + 4); + } + if (ioStream != nullptr && amountToRead > 0) { - if (request->IsEventStreamRequest()) + if (isStreaming) { if (ioStream->readsome(ptr, amountToRead) == 0 && !ioStream->eof()) { @@ -247,6 +281,39 @@ static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata) ioStream->read(ptr, amountToRead); } size_t amountRead = static_cast<size_t>(ioStream->gcount()); + + if (isAwsChunked) + { + if (amountRead > 0) + { + if (request->GetRequestHash().second != nullptr) + { + request->GetRequestHash().second->Update(reinterpret_cast<unsigned char*>(ptr), amountRead); + } + + Aws::String hex = Aws::Utils::StringUtils::ToHexString(amountRead); + memmove(ptr + hex.size() + 2, ptr, amountRead); + memmove(ptr + hex.size() + 2 + amountRead, "\r\n", 2); + memmove(ptr, hex.c_str(), hex.size()); + memmove(ptr + hex.size(), "\r\n", 2); + amountRead += hex.size() + 4; + } + else if (!context->m_chunkEnd) + { + Aws::StringStream chunkedTrailer; + chunkedTrailer << "0\r\n"; + if (request->GetRequestHash().second != nullptr) + { + chunkedTrailer << "x-amz-checksum-" << request->GetRequestHash().first << ":" + << HashingUtils::Base64Encode(request->GetRequestHash().second->GetHash().GetResult()) << "\r\n"; + } + chunkedTrailer << "\r\n"; + amountRead = chunkedTrailer.str().size(); + memcpy(ptr, chunkedTrailer.str().c_str(), amountRead); + context->m_chunkEnd = true; + } + } + auto& sentHandler = request->GetDataSentEventHandler(); if (sentHandler) { @@ -264,6 +331,14 @@ static size_t ReadBody(char* ptr, size_t size, size_t nmemb, void* userdata) return 0; } +static size_t ReadBodyStreaming(char* ptr, size_t size, size_t nmemb, void* userdata) { + return ReadBody(ptr, size, nmemb, userdata, true); +} + +static size_t ReadBodyFunc(char* ptr, size_t size, size_t nmemb, void* userdata) { + return ReadBody(ptr, size, nmemb, userdata, false); +} + static size_t SeekBody(void* userdata, curl_off_t offset, int origin) { CurlReadCallbackContext* context = reinterpret_cast<CurlReadCallbackContext*>(userdata); @@ -358,7 +433,11 @@ void SetOptCodeForHttpMethod(CURL* requestHandle, const std::shared_ptr<HttpRequ } else { +#if LIBCURL_VERSION_NUM >= 0x070c01 // 7.12.1 + curl_easy_setopt(requestHandle, CURLOPT_UPLOAD, 1L); +#else curl_easy_setopt(requestHandle, CURLOPT_PUT, 1L); +#endif } break; case HttpMethod::HTTP_HEAD: @@ -579,6 +658,9 @@ std::shared_ptr<HttpResponse> CurlHttpClient::MakeRequest(const std::shared_ptr< curl_easy_setopt(connectionHandle, CURLOPT_CAINFO, m_caFile.c_str()); } + // enable the cookie engine without reading any initial cookies. + curl_easy_setopt(connectionHandle, CURLOPT_COOKIEFILE, ""); + // only set by android test builds because the emulator is missing a cert needed for aws services #ifdef TEST_CERT_PATH curl_easy_setopt(connectionHandle, CURLOPT_CAPATH, TEST_CERT_PATH); @@ -664,12 +746,13 @@ std::shared_ptr<HttpResponse> CurlHttpClient::MakeRequest(const std::shared_ptr< if (request->GetContentBody()) { - curl_easy_setopt(connectionHandle, CURLOPT_READFUNCTION, ReadBody); + curl_easy_setopt(connectionHandle, CURLOPT_READFUNCTION, ReadBodyFunc); curl_easy_setopt(connectionHandle, CURLOPT_READDATA, &readContext); curl_easy_setopt(connectionHandle, CURLOPT_SEEKFUNCTION, SeekBody); curl_easy_setopt(connectionHandle, CURLOPT_SEEKDATA, &readContext); - if (request->IsEventStreamRequest()) + if (request->IsEventStreamRequest() && !response->HasHeader(Aws::Http::X_AMZN_ERROR_TYPE)) { + curl_easy_setopt(connectionHandle, CURLOPT_READFUNCTION, ReadBodyStreaming); curl_easy_setopt(connectionHandle, CURLOPT_NOPROGRESS, 0L); #if LIBCURL_VERSION_NUM >= 0x072000 // 7.32.0 curl_easy_setopt(connectionHandle, CURLOPT_XFERINFOFUNCTION, CurlProgressCallback); @@ -714,15 +797,18 @@ std::shared_ptr<HttpResponse> CurlHttpClient::MakeRequest(const std::shared_ptr< AWS_LOGSTREAM_DEBUG(CURL_HTTP_CLIENT_TAG, "Returned content type " << contentType); } + bool hasContentLength = false; + int64_t contentLength = + GetContentLengthFromHeader(connectionHandle, hasContentLength); + if (request->GetMethod() != HttpMethod::HTTP_HEAD && writeContext.m_client->IsRequestProcessingEnabled() && - response->HasHeader(Aws::Http::CONTENT_LENGTH_HEADER)) + hasContentLength) { - const Aws::String& contentLength = response->GetHeader(Aws::Http::CONTENT_LENGTH_HEADER); int64_t numBytesResponseReceived = writeContext.m_numBytesResponseReceived; AWS_LOGSTREAM_TRACE(CURL_HTTP_CLIENT_TAG, "Response content-length header: " << contentLength); AWS_LOGSTREAM_TRACE(CURL_HTTP_CLIENT_TAG, "Response body length: " << numBytesResponseReceived); - if (StringUtils::ConvertToInt64(contentLength.c_str()) != numBytesResponseReceived) + if (contentLength != numBytesResponseReceived) { response->SetClientErrorType(CoreErrors::NETWORK_CONNECTION); response->SetClientErrorMessage("Response body length doesn't match the content-length header."); diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpRequest.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpRequest.cpp index 47a0ee4fac..87b857ca24 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpRequest.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpRequest.cpp @@ -4,7 +4,7 @@ */ #include <aws/core/http/standard/StandardHttpRequest.h> - +#include <aws/core/utils/logging/LogMacros.h> #include <aws/core/utils/StringUtils.h> #include <iostream> @@ -15,6 +15,8 @@ using namespace Aws::Http; using namespace Aws::Http::Standard; using namespace Aws::Utils; +static const char* STANDARD_HTTP_REQUEST_LOG_TAG = "StandardHttpRequest"; + static bool IsDefaultPort(const URI& uri) { switch(uri.GetPort()) @@ -59,8 +61,13 @@ HeaderValueCollection StandardHttpRequest::GetHeaders() const const Aws::String& StandardHttpRequest::GetHeaderValue(const char* headerName) const { - auto iter = headerMap.find(headerName); + auto iter = headerMap.find(StringUtils::ToLower(headerName)); assert (iter != headerMap.end()); + if (iter == headerMap.end()) { + AWS_LOGSTREAM_ERROR(STANDARD_HTTP_REQUEST_LOG_TAG, "Requested a header value for a missing header key: " << headerName); + static const Aws::String EMPTY_STRING = ""; + return EMPTY_STRING; + } return iter->second; } diff --git a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpResponse.cpp b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpResponse.cpp index 92d7a062b6..8b62ae5e63 100644 --- a/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpResponse.cpp +++ b/contrib/libs/aws-sdk-cpp/aws-cpp-sdk-core/source/http/standard/StandardHttpResponse.cpp @@ -6,6 +6,7 @@ #include <aws/core/http/standard/StandardHttpResponse.h> #include <aws/core/utils/StringUtils.h> +#include <aws/core/utils/logging/LogMacros.h> #include <aws/core/utils/memory/AWSMemory.h> #include <istream> @@ -14,6 +15,7 @@ using namespace Aws::Http; using namespace Aws::Http::Standard; using namespace Aws::Utils; +static const char* STANDARD_HTTP_RESPONSE_LOG_TAG = "StandardHttpResponse"; HeaderValueCollection StandardHttpResponse::GetHeaders() const { @@ -35,6 +37,12 @@ bool StandardHttpResponse::HasHeader(const char* headerName) const const Aws::String& StandardHttpResponse::GetHeader(const Aws::String& headerName) const { Aws::Map<Aws::String, Aws::String>::const_iterator foundValue = headerMap.find(StringUtils::ToLower(headerName.c_str())); + assert(foundValue != headerMap.end()); + if (foundValue == headerMap.end()) { + AWS_LOGSTREAM_ERROR(STANDARD_HTTP_RESPONSE_LOG_TAG, "Requested a header value for a missing header key: " << headerName); + static const Aws::String EMPTY_STRING = ""; + return EMPTY_STRING; + } return foundValue->second; } |