diff options
author | Alexey Efimov <xeno@prnwatch.com> | 2022-02-10 16:49:41 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:41 +0300 |
commit | 26e0e4fb5e5cd6b4d7f4c21f9fcd7978891bf946 (patch) | |
tree | d34555f21d4d9f94f84d460e55b77d7eb41a953c /library/cpp/http/fetch/httpagent.h | |
parent | ca3252a147a429eac4ba8221857493c58dcd09b5 (diff) | |
download | ydb-26e0e4fb5e5cd6b4d7f4c21f9fcd7978891bf946.tar.gz |
Restoring authorship annotation for Alexey Efimov <xeno@prnwatch.com>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/http/fetch/httpagent.h')
-rw-r--r-- | library/cpp/http/fetch/httpagent.h | 630 |
1 files changed, 315 insertions, 315 deletions
diff --git a/library/cpp/http/fetch/httpagent.h b/library/cpp/http/fetch/httpagent.h index 96475cc05d..10696b6a3a 100644 --- a/library/cpp/http/fetch/httpagent.h +++ b/library/cpp/http/fetch/httpagent.h @@ -1,316 +1,316 @@ -#pragma once - -#include <cstdio> -#include <cstring> -#include <cstdlib> - -#include <library/cpp/uri/http_url.h> -#include <util/datetime/base.h> -#include <util/network/hostip.h> -#include <util/network/ip.h> -#include <util/network/sock.h> -#include <util/generic/scope.h> -#include <util/generic/utility.h> -#include <util/string/cast.h> - -#include "exthttpcodes.h" -#include "sockhandler.h" - -class TIpResolver { -public: - TAddrList Resolve(const char* host, TIpPort port) const { - try { - TAddrList result; - TNetworkAddress na(host, port); - for (auto i = na.Begin(); i != na.End(); ++i) { - const struct addrinfo& ai = *i; - switch (ai.ai_family) { - case AF_INET: - result.push_back(new NAddr::TIPv4Addr(*(sockaddr_in*)ai.ai_addr)); - break; - case AF_INET6: - result.push_back(new NAddr::TIPv6Addr(*(sockaddr_in6*)ai.ai_addr)); - break; - } - } - return result; - } catch (const TNetworkResolutionError&) { - } - return TAddrList(); - } -}; - -namespace NResolverHelpers { - Y_HAS_MEMBER(Resolve); - - template <typename TResolver> - std::enable_if_t<TClassHasResolve<TResolver>::value, TAddrList> Resolve(const TResolver& r, const char* host, TIpPort port) { - return r.Resolve(host, port); - } - - template <typename TResolver> - std::enable_if_t<!TClassHasResolve<TResolver>::value, TAddrList> Resolve(const TResolver& r, const char* host, TIpPort port) { - ui32 ip = 0; - if (r.GetHostIP(host, &ip)) { - // error - return TAddrList(); - } - if (!ip) { - return TAddrList(); - } - - return TAddrList::MakeV4Addr(ip, port); - } -} - -template <typename TBase> -class TIpResolverWrapper { -private: - TBase Base; - -public: - TIpResolverWrapper() = default; - - template <typename T> - TIpResolverWrapper(T&& base) - : Base(std::forward(base)) - { - } - - TAddrList Resolve(const char* host, TIpPort port) const { - return NResolverHelpers::Resolve(Base, host, port); - } -}; - -template <class TSocketHandler = TSimpleSocketHandler, class TDnsClient = TIpResolver> -class THttpAgent { -public: - THttpAgent() - : Persistent(0) - , Timeout(TDuration::MicroSeconds(150)) - , Hostheader(nullptr) - , Footer(nullptr) - , AltFooter(nullptr) - , PostData(nullptr) - , PostDataLen(0) - , Method(nullptr) - , MethodLen(0) - , HostheaderLen(0) - { - SetIdentification("YandexSomething/1.0", "webadmin@yandex.ru"); - } - - ~THttpAgent() { - Disconnect(); - free(Hostheader); - free(Footer); - } - - void SetIdentification(const char* user_agent, const char* http_from) { - free(Footer); - size_t len = user_agent ? strlen(user_agent) + 15 : 0; - len += http_from ? strlen(http_from) + 9 : 0; - len += 3; - Footer = (char*)malloc(len); - if (user_agent) - strcat(strcat(strcpy(Footer, "User-Agent: "), user_agent), "\r\n"); - if (http_from) - strcat(strcat(strcat(Footer, "From: "), http_from), "\r\n"); - } - - void SetUserAgentFooter(const char* altFooter) { - AltFooter = altFooter; - } - - void SetPostData(const char* postData, size_t postDataLen) { - PostData = postData; - PostDataLen = postDataLen; - } - - void SetMethod(const char* method, size_t methodLen) { - Method = method; - MethodLen = methodLen; - } - - // deprecated - ui32 GetIp() const { - return Addrs.GetV4Addr().first; - } - - int GetScheme() const { - return THttpURL::SchemeHTTP; - } - void SetTimeout(TDuration tim) { - Timeout = tim; - } - - void SetConnectTimeout(TDuration timeout) { - ConnectTimeout = timeout; - } - - int Disconnected() { - return !Persistent || !Socket.Good(); - } - - int SetHost(const char* hostname, TIpPort port) { - Disconnect(); - TAddrList addrs = DnsClient.Resolve(hostname, port); - if (!addrs.size()) { - return 1; - } - - SetHost(hostname, port, addrs); - return 0; - } - - int SetHost(const char* hostname, TIpPort port, const TAddrList& addrs) { - Disconnect(); - Addrs = addrs; - size_t reqHostheaderLen = strlen(hostname) + 20; - if (HostheaderLen < reqHostheaderLen) { - free(Hostheader); - Hostheader = (char*)malloc((HostheaderLen = reqHostheaderLen)); - } - if (port == 80) - sprintf(Hostheader, "Host: %s\r\n", hostname); - else - sprintf(Hostheader, "Host: %s:%u\r\n", hostname, port); - pHostBeg = strchr(Hostheader, ' ') + 1; - pHostEnd = strchr(pHostBeg, '\r'); - // convert hostname to lower case since some web server don't like - // uppper case (Task ROBOT-562) - for (char* p = pHostBeg; p < pHostEnd; p++) - *p = tolower(*p); - return 0; - } - - // deprecated v4-only - int SetHost(const char* hostname, TIpPort port, ui32 ip) { - return SetHost(hostname, port, TAddrList::MakeV4Addr(ip, port)); - } - - void SetHostHeader(const char* host) { - size_t reqHostheaderLen = strlen(host) + 20; - if (HostheaderLen < reqHostheaderLen) { - delete[] Hostheader; - Hostheader = new char[(HostheaderLen = reqHostheaderLen)]; - } - sprintf(Hostheader, "Host: %s\r\n", host); - } - - void SetSocket(SOCKET fd) { - Socket.SetSocket(fd); - } - - SOCKET PickOutSocket() { - return Socket.PickOutSocket(); - } - - void Disconnect() { - Socket.Disconnect(); - } - - ssize_t read(void* buffer, size_t buflen) { - return Socket.read(buffer, buflen); - } - - int RequestGet(const char* url, const char* const* headers, int persistent = 1, bool head_request = false) { - if (!Addrs.size()) - return HTTP_DNS_FAILURE; - char message[MessageMax]; - ssize_t messlen = 0; - if (Method) { - strncpy(message, Method, MethodLen); - message[MethodLen] = ' '; - messlen = MethodLen + 1; - } else if (PostData) { - strcpy(message, "POST "); - messlen = 5; - } else if (head_request) { - strcpy(message, "HEAD "); - messlen = 5; - } else { - strcpy(message, "GET "); - messlen = 4; - } -#define _AppendMessage(mes) messlen += Min(MessageMax - messlen, \ - (ssize_t)strlcpy(message + messlen, (mes), MessageMax - messlen)) - _AppendMessage(url); - _AppendMessage(" HTTP/1.1\r\n"); - if (*url == '/') //if not then Host is a proxy - _AppendMessage(Hostheader); - _AppendMessage("Connection: "); - _AppendMessage(persistent ? "Keep-Alive\r\n" : "Close\r\n"); - while (headers && *headers) - _AppendMessage(*headers++); - if (AltFooter) - _AppendMessage(AltFooter); - else - _AppendMessage(Footer); - _AppendMessage("\r\n"); -#undef _AppendMessage - if (messlen >= MessageMax) - return HTTP_HEADER_TOO_LARGE; - - if (!Persistent) - Disconnect(); - Persistent = persistent; - int connected = Socket.Good(); - for (int attempt = !connected; attempt < 2; attempt++) { - const auto connectTimeout = ConnectTimeout ? ConnectTimeout : Timeout; - if (!Socket.Good() && Socket.Connect(Addrs, connectTimeout)) - return HTTP_CONNECT_FAILED; - - int sendOk = Socket.send(message, messlen); - if (sendOk && PostData && PostDataLen) - sendOk = Socket.send(PostData, PostDataLen); - if (!sendOk) { - int err = errno; - Disconnect(); - errno = err; - continue; - } - - if (!Socket.peek()) { - int err = errno; - Disconnect(); - if (err == EINTR) { - errno = err; - return HTTP_INTERRUPTED; - } - } else { - if (!persistent) - Socket.shutdown(); - return 0; - } - } - return connected ? HTTP_CONNECTION_LOST : HTTP_CONNECT_FAILED; - } - -protected: - TSocketHandler Socket; - TIpResolverWrapper<TDnsClient> DnsClient; - TAddrList Addrs; - int Persistent; - TDuration Timeout; - TDuration ConnectTimeout; - char *Hostheader, *Footer, *pHostBeg, *pHostEnd; - const char* AltFooter; // alternative footer can be set by the caller - const char* PostData; - size_t PostDataLen; - const char* Method; - size_t MethodLen; - unsigned short HostheaderLen; +#pragma once + +#include <cstdio> +#include <cstring> +#include <cstdlib> + +#include <library/cpp/uri/http_url.h> +#include <util/datetime/base.h> +#include <util/network/hostip.h> +#include <util/network/ip.h> +#include <util/network/sock.h> +#include <util/generic/scope.h> +#include <util/generic/utility.h> +#include <util/string/cast.h> + +#include "exthttpcodes.h" +#include "sockhandler.h" + +class TIpResolver { +public: + TAddrList Resolve(const char* host, TIpPort port) const { + try { + TAddrList result; + TNetworkAddress na(host, port); + for (auto i = na.Begin(); i != na.End(); ++i) { + const struct addrinfo& ai = *i; + switch (ai.ai_family) { + case AF_INET: + result.push_back(new NAddr::TIPv4Addr(*(sockaddr_in*)ai.ai_addr)); + break; + case AF_INET6: + result.push_back(new NAddr::TIPv6Addr(*(sockaddr_in6*)ai.ai_addr)); + break; + } + } + return result; + } catch (const TNetworkResolutionError&) { + } + return TAddrList(); + } +}; + +namespace NResolverHelpers { + Y_HAS_MEMBER(Resolve); + + template <typename TResolver> + std::enable_if_t<TClassHasResolve<TResolver>::value, TAddrList> Resolve(const TResolver& r, const char* host, TIpPort port) { + return r.Resolve(host, port); + } + + template <typename TResolver> + std::enable_if_t<!TClassHasResolve<TResolver>::value, TAddrList> Resolve(const TResolver& r, const char* host, TIpPort port) { + ui32 ip = 0; + if (r.GetHostIP(host, &ip)) { + // error + return TAddrList(); + } + if (!ip) { + return TAddrList(); + } + + return TAddrList::MakeV4Addr(ip, port); + } +} + +template <typename TBase> +class TIpResolverWrapper { +private: + TBase Base; + +public: + TIpResolverWrapper() = default; + + template <typename T> + TIpResolverWrapper(T&& base) + : Base(std::forward(base)) + { + } + + TAddrList Resolve(const char* host, TIpPort port) const { + return NResolverHelpers::Resolve(Base, host, port); + } +}; + +template <class TSocketHandler = TSimpleSocketHandler, class TDnsClient = TIpResolver> +class THttpAgent { +public: + THttpAgent() + : Persistent(0) + , Timeout(TDuration::MicroSeconds(150)) + , Hostheader(nullptr) + , Footer(nullptr) + , AltFooter(nullptr) + , PostData(nullptr) + , PostDataLen(0) + , Method(nullptr) + , MethodLen(0) + , HostheaderLen(0) + { + SetIdentification("YandexSomething/1.0", "webadmin@yandex.ru"); + } + + ~THttpAgent() { + Disconnect(); + free(Hostheader); + free(Footer); + } + + void SetIdentification(const char* user_agent, const char* http_from) { + free(Footer); + size_t len = user_agent ? strlen(user_agent) + 15 : 0; + len += http_from ? strlen(http_from) + 9 : 0; + len += 3; + Footer = (char*)malloc(len); + if (user_agent) + strcat(strcat(strcpy(Footer, "User-Agent: "), user_agent), "\r\n"); + if (http_from) + strcat(strcat(strcat(Footer, "From: "), http_from), "\r\n"); + } + + void SetUserAgentFooter(const char* altFooter) { + AltFooter = altFooter; + } + + void SetPostData(const char* postData, size_t postDataLen) { + PostData = postData; + PostDataLen = postDataLen; + } + + void SetMethod(const char* method, size_t methodLen) { + Method = method; + MethodLen = methodLen; + } + + // deprecated + ui32 GetIp() const { + return Addrs.GetV4Addr().first; + } + + int GetScheme() const { + return THttpURL::SchemeHTTP; + } + void SetTimeout(TDuration tim) { + Timeout = tim; + } + + void SetConnectTimeout(TDuration timeout) { + ConnectTimeout = timeout; + } + + int Disconnected() { + return !Persistent || !Socket.Good(); + } + + int SetHost(const char* hostname, TIpPort port) { + Disconnect(); + TAddrList addrs = DnsClient.Resolve(hostname, port); + if (!addrs.size()) { + return 1; + } + + SetHost(hostname, port, addrs); + return 0; + } + + int SetHost(const char* hostname, TIpPort port, const TAddrList& addrs) { + Disconnect(); + Addrs = addrs; + size_t reqHostheaderLen = strlen(hostname) + 20; + if (HostheaderLen < reqHostheaderLen) { + free(Hostheader); + Hostheader = (char*)malloc((HostheaderLen = reqHostheaderLen)); + } + if (port == 80) + sprintf(Hostheader, "Host: %s\r\n", hostname); + else + sprintf(Hostheader, "Host: %s:%u\r\n", hostname, port); + pHostBeg = strchr(Hostheader, ' ') + 1; + pHostEnd = strchr(pHostBeg, '\r'); + // convert hostname to lower case since some web server don't like + // uppper case (Task ROBOT-562) + for (char* p = pHostBeg; p < pHostEnd; p++) + *p = tolower(*p); + return 0; + } + + // deprecated v4-only + int SetHost(const char* hostname, TIpPort port, ui32 ip) { + return SetHost(hostname, port, TAddrList::MakeV4Addr(ip, port)); + } + + void SetHostHeader(const char* host) { + size_t reqHostheaderLen = strlen(host) + 20; + if (HostheaderLen < reqHostheaderLen) { + delete[] Hostheader; + Hostheader = new char[(HostheaderLen = reqHostheaderLen)]; + } + sprintf(Hostheader, "Host: %s\r\n", host); + } + + void SetSocket(SOCKET fd) { + Socket.SetSocket(fd); + } + + SOCKET PickOutSocket() { + return Socket.PickOutSocket(); + } + + void Disconnect() { + Socket.Disconnect(); + } + + ssize_t read(void* buffer, size_t buflen) { + return Socket.read(buffer, buflen); + } + + int RequestGet(const char* url, const char* const* headers, int persistent = 1, bool head_request = false) { + if (!Addrs.size()) + return HTTP_DNS_FAILURE; + char message[MessageMax]; + ssize_t messlen = 0; + if (Method) { + strncpy(message, Method, MethodLen); + message[MethodLen] = ' '; + messlen = MethodLen + 1; + } else if (PostData) { + strcpy(message, "POST "); + messlen = 5; + } else if (head_request) { + strcpy(message, "HEAD "); + messlen = 5; + } else { + strcpy(message, "GET "); + messlen = 4; + } +#define _AppendMessage(mes) messlen += Min(MessageMax - messlen, \ + (ssize_t)strlcpy(message + messlen, (mes), MessageMax - messlen)) + _AppendMessage(url); + _AppendMessage(" HTTP/1.1\r\n"); + if (*url == '/') //if not then Host is a proxy + _AppendMessage(Hostheader); + _AppendMessage("Connection: "); + _AppendMessage(persistent ? "Keep-Alive\r\n" : "Close\r\n"); + while (headers && *headers) + _AppendMessage(*headers++); + if (AltFooter) + _AppendMessage(AltFooter); + else + _AppendMessage(Footer); + _AppendMessage("\r\n"); +#undef _AppendMessage + if (messlen >= MessageMax) + return HTTP_HEADER_TOO_LARGE; + + if (!Persistent) + Disconnect(); + Persistent = persistent; + int connected = Socket.Good(); + for (int attempt = !connected; attempt < 2; attempt++) { + const auto connectTimeout = ConnectTimeout ? ConnectTimeout : Timeout; + if (!Socket.Good() && Socket.Connect(Addrs, connectTimeout)) + return HTTP_CONNECT_FAILED; + + int sendOk = Socket.send(message, messlen); + if (sendOk && PostData && PostDataLen) + sendOk = Socket.send(PostData, PostDataLen); + if (!sendOk) { + int err = errno; + Disconnect(); + errno = err; + continue; + } + + if (!Socket.peek()) { + int err = errno; + Disconnect(); + if (err == EINTR) { + errno = err; + return HTTP_INTERRUPTED; + } + } else { + if (!persistent) + Socket.shutdown(); + return 0; + } + } + return connected ? HTTP_CONNECTION_LOST : HTTP_CONNECT_FAILED; + } + +protected: + TSocketHandler Socket; + TIpResolverWrapper<TDnsClient> DnsClient; + TAddrList Addrs; + int Persistent; + TDuration Timeout; + TDuration ConnectTimeout; + char *Hostheader, *Footer, *pHostBeg, *pHostEnd; + const char* AltFooter; // alternative footer can be set by the caller + const char* PostData; + size_t PostDataLen; + const char* Method; + size_t MethodLen; + unsigned short HostheaderLen; static const ssize_t MessageMax = 32768; -}; - -struct TNoTimer { - inline void OnBeforeSend() { - } - inline void OnAfterSend() { - } - inline void OnBeforeRecv() { - } - inline void OnAfterRecv() { - } -}; +}; + +struct TNoTimer { + inline void OnBeforeSend() { + } + inline void OnAfterSend() { + } + inline void OnBeforeRecv() { + } + inline void OnAfterRecv() { + } +}; |