diff options
author | lonelly <lonelly@yandex-team.ru> | 2022-02-10 16:49:49 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:49:49 +0300 |
commit | e01af096d5cd7baca583319f1473a2a2c993ec86 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /library/cpp/http | |
parent | f1a86949d52e1185ed4f05de191b50386abc68f4 (diff) | |
download | ydb-e01af096d5cd7baca583319f1473a2a2c993ec86.tar.gz |
Restoring authorship annotation for <lonelly@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/http')
-rw-r--r-- | library/cpp/http/fetch/httpfsm.h | 26 | ||||
-rw-r--r-- | library/cpp/http/fetch/httpfsm.rl6 | 406 | ||||
-rw-r--r-- | library/cpp/http/fetch/httpfsm_ut.cpp | 232 | ||||
-rw-r--r-- | library/cpp/http/fetch/httpheader.h | 224 |
4 files changed, 444 insertions, 444 deletions
diff --git a/library/cpp/http/fetch/httpfsm.h b/library/cpp/http/fetch/httpfsm.h index 18c00893a6..c4abdcd0d2 100644 --- a/library/cpp/http/fetch/httpfsm.h +++ b/library/cpp/http/fetch/httpfsm.h @@ -24,12 +24,12 @@ struct THttpHeaderParser { } int Init(THttpHeader* h) { - int ret = Init((THttpBaseHeader*)(h)); + int ret = Init((THttpBaseHeader*)(h)); hd = h; hd->Init(); hreflangpos = hd->hreflangs; hreflangspace = HREFLANG_MAX; - return ret; + return ret; } int Init(THttpAuthHeader* h) { @@ -38,11 +38,11 @@ struct THttpHeaderParser { return ret; } int Init(THttpRequestHeader* h) { - int ret = Init((THttpBaseHeader*)(h)); - request_hd = h; - request_hd->Init(); - return ret; - } + int ret = Init((THttpBaseHeader*)(h)); + request_hd = h; + request_hd->Init(); + return ret; + } THttpHeader* hd; long I; @@ -67,16 +67,16 @@ private: THttpBaseHeader* base_hd; int cs; -private: +private: int Init(THttpBaseHeader* header) { - base_hd = header; + base_hd = header; auth_hd = nullptr; request_hd = nullptr; hd = nullptr; - init(); - return 0; - } - + init(); + return 0; + } + int execute(unsigned char* inBuf, int len); void init(); }; diff --git a/library/cpp/http/fetch/httpfsm.rl6 b/library/cpp/http/fetch/httpfsm.rl6 index ca8458bd12..eab0328b18 100644 --- a/library/cpp/http/fetch/httpfsm.rl6 +++ b/library/cpp/http/fetch/httpfsm.rl6 @@ -65,60 +65,60 @@ action clear_buf { buflen = 0; } action update_buf { if (buflen < sizeof(buf)) buf[buflen++] = fc; } ################################################### -############ response status line ################# -action set_minor { base_hd->http_minor = I; } -action set_status { - if (hd) { - hd->http_status = I; - } - if (request_hd) { - return -3; - } -} - -status_code = int3; -http_major = int; -http_minor = int; -reason_phrase = ws+ text_char*; -http_version = "http/"i http_major '.' http_minor %set_minor; -response_status_line = http_version ws+ status_code reason_phrase? eol %set_status; - -############ request status line ################# -action set_request_uri { +############ response status line ################# +action set_minor { base_hd->http_minor = I; } +action set_status { + if (hd) { + hd->http_status = I; + } + if (request_hd) { + return -3; + } +} + +status_code = int3; +http_major = int; +http_minor = int; +reason_phrase = ws+ text_char*; +http_version = "http/"i http_major '.' http_minor %set_minor; +response_status_line = http_version ws+ status_code reason_phrase? eol %set_status; + +############ request status line ################# +action set_request_uri { if (request_hd && buflen < FETCHER_URL_MAX) { if (!request_hd->request_uri.empty()) { - return -2; - } + return -2; + } request_hd->request_uri =TStringBuf(buf, buflen); - } -} -action set_http_method { - if (request_hd) { - request_hd->http_method = I; - } - if (hd) { - return -3; - } -} - -http_extension_method = token; -http_method = ("options"i %{c(0)} @1 - | "get"i %{c(1)} @1 - | "head"i %{c(2)} @1 - | "post"i %{c(3)} @1 - | "put"i %{c(4)} @1 - | "delete"i %{c(5)} @1 - | "trace"i %{c(6)} @1 - | "connect"i %{c(7)} @1 - | http_extension_method %{c(8)} $0) - %set_http_method; -request_uri = (token_char | separator)+ >clear_buf $update_buf - %set_request_uri; -request_status_line = http_method ws+ request_uri ws+ http_version eoh; - + } +} +action set_http_method { + if (request_hd) { + request_hd->http_method = I; + } + if (hd) { + return -3; + } +} + +http_extension_method = token; +http_method = ("options"i %{c(0)} @1 + | "get"i %{c(1)} @1 + | "head"i %{c(2)} @1 + | "post"i %{c(3)} @1 + | "put"i %{c(4)} @1 + | "delete"i %{c(5)} @1 + | "trace"i %{c(6)} @1 + | "connect"i %{c(7)} @1 + | http_extension_method %{c(8)} $0) + %set_http_method; +request_uri = (token_char | separator)+ >clear_buf $update_buf + %set_request_uri; +request_status_line = http_method ws+ request_uri ws+ http_version eoh; + ################# connection ###################### -action beg_connection { guard(base_hd->connection_closed); I = -1; } -action set_connection { setguarded(base_hd->connection_closed, I); } +action beg_connection { guard(base_hd->connection_closed); I = -1; } +action set_connection { setguarded(base_hd->connection_closed, I); } c_token = "close"i %{m(1)} | "keep-alive"i %{m(0)}; @@ -127,7 +127,7 @@ connection = "connection"i def %beg_connection c_tokenlist eoh %set_connect ################# content-encoding ################ action beg_content_encoding { I = HTTP_COMPRESSION_ERROR; } -action set_content_encoding { base_hd->compression_method = +action set_content_encoding { base_hd->compression_method = ((base_hd->compression_method == HTTP_COMPRESSION_UNSET || base_hd->compression_method == I) ? I : (int)HTTP_COMPRESSION_ERROR); } @@ -141,51 +141,51 @@ ce_tokenlist = "identity"i %{c(HTTP_COMPRESSION_IDENTITY)} content_encoding = "content-encoding"i def %beg_content_encoding ce_tokenlist eoh %set_content_encoding; ################# transfer-encoding ############### -action beg_encoding { guard(base_hd->transfer_chunked); } -action set_encoding { setguarded(base_hd->transfer_chunked, I); } +action beg_encoding { guard(base_hd->transfer_chunked); } +action set_encoding { setguarded(base_hd->transfer_chunked, I); } e_tokenlist = "identity"i %{c(0)} | "chunked"i %{c(1)}; transfer_encoding = "transfer-encoding"i def %beg_encoding e_tokenlist eoh %set_encoding; ################# content-length ################## -action beg_content_length { guard(base_hd->content_length); } -action set_content_length { setguarded(base_hd->content_length, I); } +action beg_content_length { guard(base_hd->content_length); } +action set_content_length { setguarded(base_hd->content_length, I); } content_length = "content-length"i def %beg_content_length int eoh %set_content_length; ################# content-range ################### -action beg_content_range_start { guard(base_hd->content_range_start); I = -1; } -action set_content_range_start { setguarded(base_hd->content_range_start, I); } -action beg_content_range_end { guard(base_hd->content_range_end); I = -1; } -action set_content_range_end { setguarded(base_hd->content_range_end, I); } -action beg_content_range_el { guard(base_hd->content_range_entity_length); I = -1; } -action set_content_range_el { setguarded(base_hd->content_range_entity_length, I); } +action beg_content_range_start { guard(base_hd->content_range_start); I = -1; } +action set_content_range_start { setguarded(base_hd->content_range_start, I); } +action beg_content_range_end { guard(base_hd->content_range_end); I = -1; } +action set_content_range_end { setguarded(base_hd->content_range_end, I); } +action beg_content_range_el { guard(base_hd->content_range_entity_length); I = -1; } +action set_content_range_el { setguarded(base_hd->content_range_entity_length, I); } content_range = "content-range"i def "bytes"i sp %beg_content_range_start int '-' %set_content_range_start %beg_content_range_end int '/' %set_content_range_end %beg_content_range_el int eoh %set_content_range_el; ################# accept-ranges ################### -action beg_accept_ranges { - if (hd) { - guard(hd->accept_ranges); - I = -1; - } -} -action set_accept_ranges { if (hd) setguarded(hd->accept_ranges, I); } +action beg_accept_ranges { + if (hd) { + guard(hd->accept_ranges); + I = -1; + } +} +action set_accept_ranges { if (hd) setguarded(hd->accept_ranges, I); } ar_tokenlist = "bytes"i %{c(1)} | "none"i %{c(0)}; accept_ranges = "accept-ranges"i def %beg_accept_ranges ar_tokenlist eoh %set_accept_ranges; ################# content-type #################### -action beg_mime { guard(base_hd->mime_type); } -action set_mime { setguarded(base_hd->mime_type, I); } +action beg_mime { guard(base_hd->mime_type); } +action set_mime { setguarded(base_hd->mime_type, I); } action set_charset { if (buflen < FETCHER_URL_MAX) { buf[buflen++] = 0; - base_hd->charset = EncodingHintByName((const char*)buf); + base_hd->charset = EncodingHintByName((const char*)buf); } } @@ -240,10 +240,10 @@ mime_parms = (lws ';' lws mime_param)*; content_type = "content-type"i def %beg_mime mime_type mime_parms eoh %set_mime; ################# last modified ################### -action beg_modtime { guard(base_hd->http_time); } -action set_modtime { - setguarded(base_hd->http_time, DateTimeFields.ToTimeT(-1)); -} +action beg_modtime { guard(base_hd->http_time); } +action set_modtime { + setguarded(base_hd->http_time, DateTimeFields.ToTimeT(-1)); +} last_modified = "last-modified"i def %beg_modtime http_date eoh %set_modtime; @@ -257,7 +257,7 @@ action set_location { } } -action set_status_303{ if (hd) hd->http_status = 303; } +action set_status_303{ if (hd) hd->http_status = 303; } url = url_char+ >clear_buf $update_buf; loc_url = any_text_char+ >clear_buf $update_buf; @@ -399,48 +399,48 @@ digest_challenge = ("digest"i %init_auth ws+ auth_params) | auth = "www-authenticate"i def digest_challenge eoh; -###################### host ####################### -action set_host { - if (request_hd && buflen < HOST_MAX) { - buf[buflen++] = 0; - if (request_hd->host[0] != 0) { - return -2; - } - memcpy(request_hd->host, buf, buflen); - } -} - -host = (url_char | [:])* >clear_buf $update_buf; -host_header = "host"i def host eoh %set_host; - -###################### from ####################### -action set_from { - if (request_hd && buflen < MAXWORD_LEN) { - buf[buflen++] = 0; - if (request_hd->from[0] != 0) { - return -2; - } - memcpy(request_hd->from, buf, buflen); - } -} - -mailbox = (token "@" token) >clear_buf $update_buf; -from_header = "from"i def mailbox eoh %set_from; - -################### user-agent #################### -action set_user_agent { - if (request_hd && buflen < MAXWORD_LEN) { - buf[buflen++] = 0; - if (request_hd->user_agent[0] != 0) { - return -2; - } - memcpy(request_hd->user_agent, buf, buflen); - } -} - -user_agent = any_text_char* >clear_buf $update_buf; -user_agent_header = "user-agent"i def user_agent eoh %set_user_agent; - +###################### host ####################### +action set_host { + if (request_hd && buflen < HOST_MAX) { + buf[buflen++] = 0; + if (request_hd->host[0] != 0) { + return -2; + } + memcpy(request_hd->host, buf, buflen); + } +} + +host = (url_char | [:])* >clear_buf $update_buf; +host_header = "host"i def host eoh %set_host; + +###################### from ####################### +action set_from { + if (request_hd && buflen < MAXWORD_LEN) { + buf[buflen++] = 0; + if (request_hd->from[0] != 0) { + return -2; + } + memcpy(request_hd->from, buf, buflen); + } +} + +mailbox = (token "@" token) >clear_buf $update_buf; +from_header = "from"i def mailbox eoh %set_from; + +################### user-agent #################### +action set_user_agent { + if (request_hd && buflen < MAXWORD_LEN) { + buf[buflen++] = 0; + if (request_hd->user_agent[0] != 0) { + return -2; + } + memcpy(request_hd->user_agent, buf, buflen); + } +} + +user_agent = any_text_char* >clear_buf $update_buf; +user_agent_header = "user-agent"i def user_agent eoh %set_user_agent; + ############### x-yandex-langregion ################ action set_langregion { if (request_hd && buflen < MAX_LANGREGION_LEN) { @@ -455,20 +455,20 @@ action set_langregion { langregion = any_text_char* >clear_buf $update_buf; langregion_header = "x-yandex-langregion"i def langregion eoh %set_langregion; -############### x-yandex-sourcename ################ -action set_sourcename { - if (request_hd && buflen < MAXWORD_LEN) { - buf[buflen++] = 0; - if (request_hd->x_yandex_sourcename[0] != 0) { - return -2; - } - memcpy(request_hd->x_yandex_sourcename, buf, buflen); - } -} - -sourcename = any_text_char* >clear_buf $update_buf; -sourcename_header = "x-yandex-sourcename"i def sourcename eoh %set_sourcename; - +############### x-yandex-sourcename ################ +action set_sourcename { + if (request_hd && buflen < MAXWORD_LEN) { + buf[buflen++] = 0; + if (request_hd->x_yandex_sourcename[0] != 0) { + return -2; + } + memcpy(request_hd->x_yandex_sourcename, buf, buflen); + } +} + +sourcename = any_text_char* >clear_buf $update_buf; +sourcename_header = "x-yandex-sourcename"i def sourcename eoh %set_sourcename; + ############### x-yandex-requesttype ############### action set_requesttype { if (request_hd && buflen < MAXWORD_LEN) { @@ -497,16 +497,16 @@ action set_fetchoptions { fetchoptions = any_text_char* >clear_buf $update_buf; fetchoptions_header = "x-yandex-fetchoptions"i def fetchoptions eoh %set_fetchoptions; -################ if-modified-since ################ -action set_if_modified_since { - if (request_hd) { - request_hd->if_modified_since = DateTimeFields.ToTimeT(-1); - } -} - -if_modified_since = "if-modified-since"i def http_date eoh - %set_if_modified_since; - +################ if-modified-since ################ +action set_if_modified_since { + if (request_hd) { + request_hd->if_modified_since = DateTimeFields.ToTimeT(-1); + } +} + +if_modified_since = "if-modified-since"i def http_date eoh + %set_if_modified_since; + ################ retry-after ################ action set_retry_after_withdate { if (hd) { @@ -525,57 +525,57 @@ retry_after_withdate = "retry-after"i def http_date eoh retry_after_withdelta = "retry-after"i def int eoh %set_retry_after_withdelta; -############## request-cache-control ############## -action SETMAXAGE { if (request_hd) request_hd->max_age = I; } - -delta_seconds = int; -cache_extension = token ("=" (token | quoted_str))?; -request_cache_directive = "no-cache"i - | "no-store"i - | ("max-age"i "=" delta_seconds %SETMAXAGE) - | ("max-stale"i ("=" delta_seconds)?) - | ("min-fresh"i "=" delta_seconds) - | "non-transform"i - | "only-if-cached"i - | cache_extension; -request_cache_control = "cache-control"i def request_cache_directive eoh; - -############ x-yandex-response-timeout ############# - -action set_response_timeout { - if (request_hd) { - request_hd->x_yandex_response_timeout = I; - } -} - -response_timeout = "x-yandex-response-timeout"i def int eoh - %set_response_timeout; - -############ x-yandex-request-priority ############# - -action set_request_priority { - if (request_hd) { - request_hd->x_yandex_request_priority = I; - } -} - -request_priority = "x-yandex-request-priority"i def int eoh - %set_request_priority; - +############## request-cache-control ############## +action SETMAXAGE { if (request_hd) request_hd->max_age = I; } + +delta_seconds = int; +cache_extension = token ("=" (token | quoted_str))?; +request_cache_directive = "no-cache"i + | "no-store"i + | ("max-age"i "=" delta_seconds %SETMAXAGE) + | ("max-stale"i ("=" delta_seconds)?) + | ("min-fresh"i "=" delta_seconds) + | "non-transform"i + | "only-if-cached"i + | cache_extension; +request_cache_control = "cache-control"i def request_cache_directive eoh; + +############ x-yandex-response-timeout ############# + +action set_response_timeout { + if (request_hd) { + request_hd->x_yandex_response_timeout = I; + } +} + +response_timeout = "x-yandex-response-timeout"i def int eoh + %set_response_timeout; + +############ x-yandex-request-priority ############# + +action set_request_priority { + if (request_hd) { + request_hd->x_yandex_request_priority = I; + } +} + +request_priority = "x-yandex-request-priority"i def int eoh + %set_request_priority; + ################# message header ################## other_header = ( ex_token - "www-authenticate"i ) def any_text eoh; -message_header = other_header $0 - | connection @1 - | content_encoding @1 - | transfer_encoding @1 - | content_length @1 - | content_type @1 - | last_modified @1 - | refresh @1 - | content_range @1; -response_header = message_header $0 - | auth @1 - | accept_ranges @1 +message_header = other_header $0 + | connection @1 + | content_encoding @1 + | transfer_encoding @1 + | content_length @1 + | content_type @1 + | last_modified @1 + | refresh @1 + | content_range @1; +response_header = message_header $0 + | auth @1 + | accept_ranges @1 | location @1 | x_robots_tag @1 | rel_canonical @1 @@ -583,25 +583,25 @@ response_header = message_header $0 | squid_error @1 | retry_after_withdate @1 | retry_after_withdelta @1; -request_header = message_header $0 - | from_header @1 - | host_header @1 - | user_agent_header @1 - | sourcename_header @1 +request_header = message_header $0 + | from_header @1 + | host_header @1 + | user_agent_header @1 + | sourcename_header @1 | requesttype_header @1 | langregion_header @1 | fetchoptions_header @1 - | if_modified_since @1 - | request_cache_control @1 - | response_timeout @1 - | request_priority @1; + | if_modified_since @1 + | request_cache_control @1 + | response_timeout @1 + | request_priority @1; ################# main ############################ action accepted { lastchar = (char*)fpc; return 2; } -main := ((response_status_line ('\r'? response_header)*) - | (request_status_line ('\r' ? request_header)*)) - eol @accepted; +main := ((response_status_line ('\r'? response_header)*) + | (request_status_line ('\r' ? request_header)*)) + eol @accepted; }%% diff --git a/library/cpp/http/fetch/httpfsm_ut.cpp b/library/cpp/http/fetch/httpfsm_ut.cpp index eb8fba7821..b018e80101 100644 --- a/library/cpp/http/fetch/httpfsm_ut.cpp +++ b/library/cpp/http/fetch/httpfsm_ut.cpp @@ -6,8 +6,8 @@ #include <library/cpp/charset/doccodes.h> #include <library/cpp/testing/unittest/registar.h> -class THttpHeaderParserTestSuite: public TTestBase { - UNIT_TEST_SUITE(THttpHeaderParserTestSuite); +class THttpHeaderParserTestSuite: public TTestBase { + UNIT_TEST_SUITE(THttpHeaderParserTestSuite); UNIT_TEST(TestRequestHeader); UNIT_TEST(TestSplitRequestHeader); UNIT_TEST(TestTrailingData); @@ -30,25 +30,25 @@ class THttpHeaderParserTestSuite: public TTestBase { UNIT_TEST(TestHreflangOnLongInput); UNIT_TEST(TestMimeType); UNIT_TEST(TestRepeatedContentEncoding); - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE_END(); -private: +private: THolder<THttpHeaderParser> httpHeaderParser; -private: - void TestStart(); - void TestFinish(); +private: + void TestStart(); + void TestFinish(); -public: - void TestRequestHeader(); +public: + void TestRequestHeader(); void TestSplitRequestHeader(); void TestTrailingData(); - void TestProxyRequestHeader(); - void TestIncorrectRequestHeader(); - void TestLastModified(); - void TestLastModifiedCorrupted(); - void TestResponseHeaderOnRequest(); - void TestRequestHeaderOnResponse(); + void TestProxyRequestHeader(); + void TestIncorrectRequestHeader(); + void TestLastModified(); + void TestLastModifiedCorrupted(); + void TestResponseHeaderOnRequest(); + void TestRequestHeaderOnResponse(); void TestXRobotsTagUnknownTags(); void TestXRobotsTagMyBot(); void TestXRobotsTagOtherBot(); @@ -62,41 +62,41 @@ public: void TestHreflangOnLongInput(); void TestMimeType(); void TestRepeatedContentEncoding(); -}; +}; -void THttpHeaderParserTestSuite::TestStart() { +void THttpHeaderParserTestSuite::TestStart() { httpHeaderParser.Reset(new THttpHeaderParser()); } - -void THttpHeaderParserTestSuite::TestFinish() { + +void THttpHeaderParserTestSuite::TestFinish() { httpHeaderParser.Reset(); -} - -void THttpHeaderParserTestSuite::TestRequestHeader() { - TestStart(); - THttpRequestHeader httpRequestHeader; - httpHeaderParser->Init(&httpRequestHeader); +} + +void THttpHeaderParserTestSuite::TestRequestHeader() { + TestStart(); + THttpRequestHeader httpRequestHeader; + httpHeaderParser->Init(&httpRequestHeader); const char* request = "GET /search?q=hi HTTP/1.1\r\n" "Host: www.google.ru:8080\r\n\r\n"; - i32 result = httpHeaderParser->Execute(request, strlen(request)); - UNIT_ASSERT_EQUAL(result, 2); - UNIT_ASSERT_EQUAL(httpRequestHeader.http_method, HTTP_METHOD_GET); - UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.host, "www.google.ru:8080"), 0); + i32 result = httpHeaderParser->Execute(request, strlen(request)); + UNIT_ASSERT_EQUAL(result, 2); + UNIT_ASSERT_EQUAL(httpRequestHeader.http_method, HTTP_METHOD_GET); + UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.host, "www.google.ru:8080"), 0); UNIT_ASSERT_EQUAL(httpRequestHeader.request_uri, "/search?q=hi"); UNIT_ASSERT_EQUAL(httpRequestHeader.GetUrl(), "http://www.google.ru:8080/search?q=hi"); - UNIT_ASSERT_EQUAL(httpHeaderParser->lastchar - request + 1, + UNIT_ASSERT_EQUAL(httpHeaderParser->lastchar - request + 1, (i32)strlen(request)); - UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_response_timeout, + UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_response_timeout, DEFAULT_RESPONSE_TIMEOUT); - UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_request_priority, + UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_request_priority, DEFAULT_REQUEST_PRIORITY); - UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_sourcename, ""), 0); + UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_sourcename, ""), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_requesttype, ""), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_fetchoptions, ""), 0); - TestFinish(); - UNIT_ASSERT_EQUAL(httpRequestHeader.max_age, DEFAULT_MAX_AGE); -} - + TestFinish(); + UNIT_ASSERT_EQUAL(httpRequestHeader.max_age, DEFAULT_MAX_AGE); +} + void THttpHeaderParserTestSuite::TestSplitRequestHeader() { TestStart(); const char* request = @@ -152,85 +152,85 @@ void THttpHeaderParserTestSuite::TestTrailingData() { TestFinish(); } -void THttpHeaderParserTestSuite::TestProxyRequestHeader() { - TestStart(); - THttpRequestHeader httpRequestHeader; - httpHeaderParser->Init(&httpRequestHeader); +void THttpHeaderParserTestSuite::TestProxyRequestHeader() { + TestStart(); + THttpRequestHeader httpRequestHeader; + httpHeaderParser->Init(&httpRequestHeader); const char* request = - "GET http://www.google.ru:8080/search?q=hi HTTP/1.1\r\n" - "X-Yandex-Response-Timeout: 1000\r\n" - "X-Yandex-Request-Priority: 2\r\n" - "X-Yandex-Sourcename: orange\r\n" + "GET http://www.google.ru:8080/search?q=hi HTTP/1.1\r\n" + "X-Yandex-Response-Timeout: 1000\r\n" + "X-Yandex-Request-Priority: 2\r\n" + "X-Yandex-Sourcename: orange\r\n" "X-Yandex-Requesttype: userproxy\r\n" "X-Yandex-FetchOptions: d;c\r\n" - "Cache-control: max-age=100\r\n" - "If-Modified-Since: Sat, 29 Oct 1994 19:43:31 GMT\r\n" - "User-Agent: Yandex/1.01.001 (compatible; Win16; I)\r\n" - "From: webadmin@yandex.ru\r\n\r\n"; - i32 result = httpHeaderParser->Execute(request, strlen(request)); - UNIT_ASSERT_EQUAL(result, 2); - UNIT_ASSERT_EQUAL(httpRequestHeader.http_method, HTTP_METHOD_GET); - UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_response_timeout, 1000); - UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_request_priority, 2); + "Cache-control: max-age=100\r\n" + "If-Modified-Since: Sat, 29 Oct 1994 19:43:31 GMT\r\n" + "User-Agent: Yandex/1.01.001 (compatible; Win16; I)\r\n" + "From: webadmin@yandex.ru\r\n\r\n"; + i32 result = httpHeaderParser->Execute(request, strlen(request)); + UNIT_ASSERT_EQUAL(result, 2); + UNIT_ASSERT_EQUAL(httpRequestHeader.http_method, HTTP_METHOD_GET); + UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_response_timeout, 1000); + UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_request_priority, 2); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_sourcename, "orange"), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_requesttype, "userproxy"), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_fetchoptions, "d;c"), 0); - UNIT_ASSERT_EQUAL(httpRequestHeader.max_age, 100); - UNIT_ASSERT_VALUES_EQUAL(httpRequestHeader.if_modified_since, + UNIT_ASSERT_EQUAL(httpRequestHeader.max_age, 100); + UNIT_ASSERT_VALUES_EQUAL(httpRequestHeader.if_modified_since, TInstant::ParseIso8601Deprecated("1994-10-29 19:43:31Z").TimeT()); UNIT_ASSERT_EQUAL(httpRequestHeader.request_uri, "http://www.google.ru:8080/search?q=hi"); UNIT_ASSERT(httpRequestHeader.GetUrl() == "http://www.google.ru:8080/search?q=hi"); - UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.host, ""), 0); - UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.from, "webadmin@yandex.ru"), 0); - UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.user_agent, + UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.host, ""), 0); + UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.from, "webadmin@yandex.ru"), 0); + UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.user_agent, "Yandex/1.01.001 (compatible; Win16; I)"), 0); - UNIT_ASSERT_EQUAL(httpHeaderParser->lastchar - request + 1, + UNIT_ASSERT_EQUAL(httpHeaderParser->lastchar - request + 1, (i32)strlen(request)); - TestFinish(); -} - -void THttpHeaderParserTestSuite::TestIncorrectRequestHeader() { - TestStart(); - THttpRequestHeader httpRequestHeader; - httpHeaderParser->Init(&httpRequestHeader); + TestFinish(); +} + +void THttpHeaderParserTestSuite::TestIncorrectRequestHeader() { + TestStart(); + THttpRequestHeader httpRequestHeader; + httpHeaderParser->Init(&httpRequestHeader); const char* request = "GET /search?q=hi HTP/1.1\r\n" "Host: www.google.ru:8080\r\n\r\n"; - i32 result = httpHeaderParser->Execute(request, strlen(request)); - UNIT_ASSERT(result != 2); - TestFinish(); -} - -void THttpHeaderParserTestSuite::TestLastModified() { - TestStart(); - THttpHeader h; - UNIT_ASSERT(0 == httpHeaderParser->Init(&h)); - const char* headers = - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/html\r\n" - "Last-Modified: Thu, 13 Aug 2009 14:27:08 GMT\r\n\r\n"; - UNIT_ASSERT(2 == httpHeaderParser->Execute(headers, strlen(headers))); - UNIT_ASSERT_VALUES_EQUAL( + i32 result = httpHeaderParser->Execute(request, strlen(request)); + UNIT_ASSERT(result != 2); + TestFinish(); +} + +void THttpHeaderParserTestSuite::TestLastModified() { + TestStart(); + THttpHeader h; + UNIT_ASSERT(0 == httpHeaderParser->Init(&h)); + const char* headers = + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html\r\n" + "Last-Modified: Thu, 13 Aug 2009 14:27:08 GMT\r\n\r\n"; + UNIT_ASSERT(2 == httpHeaderParser->Execute(headers, strlen(headers))); + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseIso8601Deprecated("2009-08-13 14:27:08Z").TimeT(), h.http_time); - TestFinish(); -} - -void THttpHeaderParserTestSuite::TestLastModifiedCorrupted() { - TestStart(); - THttpHeader h; - UNIT_ASSERT(0 == httpHeaderParser->Init(&h)); - const char* headers = - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/html\r\n" - "Last-Modified: Thu, 13 Aug 2009 14:\r\n\r\n"; - UNIT_ASSERT(2 == httpHeaderParser->Execute(headers, strlen(headers))); - UNIT_ASSERT(h.http_time < 0); // XXX: don't understand what is the proper value - TestFinish(); -} - + TestFinish(); +} + +void THttpHeaderParserTestSuite::TestLastModifiedCorrupted() { + TestStart(); + THttpHeader h; + UNIT_ASSERT(0 == httpHeaderParser->Init(&h)); + const char* headers = + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html\r\n" + "Last-Modified: Thu, 13 Aug 2009 14:\r\n\r\n"; + UNIT_ASSERT(2 == httpHeaderParser->Execute(headers, strlen(headers))); + UNIT_ASSERT(h.http_time < 0); // XXX: don't understand what is the proper value + TestFinish(); +} + void THttpHeaderParserTestSuite::TestXRobotsTagUnknownTags() { TestStart(); THttpHeader httpHeader; @@ -424,29 +424,29 @@ void THttpHeaderParserTestSuite::TestRelCanonical() { TestFinish(); } -void THttpHeaderParserTestSuite::TestResponseHeaderOnRequest() { - TestStart(); - THttpHeader httpHeader; - httpHeaderParser->Init(&httpHeader); +void THttpHeaderParserTestSuite::TestResponseHeaderOnRequest() { + TestStart(); + THttpHeader httpHeader; + httpHeaderParser->Init(&httpHeader); const char* request = "GET /search?q=hi HTP/1.1\r\n" "Host: www.google.ru:8080\r\n\r\n"; - i32 result = httpHeaderParser->Execute(request, strlen(request)); + i32 result = httpHeaderParser->Execute(request, strlen(request)); UNIT_ASSERT_EQUAL(result, -3); - TestFinish(); -} - -void THttpHeaderParserTestSuite::TestRequestHeaderOnResponse() { - TestStart(); - THttpRequestHeader httpRequestHeader; - httpHeaderParser->Init(&httpRequestHeader); + TestFinish(); +} + +void THttpHeaderParserTestSuite::TestRequestHeaderOnResponse() { + TestStart(); + THttpRequestHeader httpRequestHeader; + httpHeaderParser->Init(&httpRequestHeader); const char* response = "HTTP/1.1 200 OK\r\n" "Content-Type: text/html\r\n" "Last-Modified: Thu, 13 Aug 2009 14:\r\n\r\n"; - i32 result = httpHeaderParser->Execute(response, strlen(response)); + i32 result = httpHeaderParser->Execute(response, strlen(response)); UNIT_ASSERT_EQUAL(result, -3); - TestFinish(); -} - + TestFinish(); +} + void THttpHeaderParserTestSuite::TestMimeType() { TestStart(); THttpHeader httpHeader; @@ -486,8 +486,8 @@ void THttpHeaderParserTestSuite::TestRepeatedContentEncoding() { TestFinish(); } -UNIT_TEST_SUITE_REGISTRATION(THttpHeaderParserTestSuite); - +UNIT_TEST_SUITE_REGISTRATION(THttpHeaderParserTestSuite); + Y_UNIT_TEST_SUITE(TestHttpChunkParser) { static THttpChunkParser initParser() { THttpChunkParser parser; diff --git a/library/cpp/http/fetch/httpheader.h b/library/cpp/http/fetch/httpheader.h index bbc39595f6..b2810bbd41 100644 --- a/library/cpp/http/fetch/httpheader.h +++ b/library/cpp/http/fetch/httpheader.h @@ -25,7 +25,7 @@ extern const i32 DEFAULT_MAX_AGE; /// == -1 extern const i8 DEFAULT_REQUEST_PRIORITY; /// == -1 extern const i32 DEFAULT_RESPONSE_TIMEOUT; /// == -1 -#define HTTP_PREFIX "http://" +#define HTTP_PREFIX "http://" #define MAX_LANGREGION_LEN 4 #define MAXWORD_LEN 55 @@ -39,28 +39,28 @@ enum HTTP_COMPRESSION { HTTP_COMPRESSION_MAX = 6 }; -enum HTTP_METHOD { - HTTP_METHOD_UNDEFINED = -1, - HTTP_METHOD_OPTIONS, - HTTP_METHOD_GET, - HTTP_METHOD_HEAD, - HTTP_METHOD_POST, - HTTP_METHOD_PUT, - HTTP_METHOD_DELETE, - HTTP_METHOD_TRACE, - HTTP_METHOD_CONNECT, - HTTP_METHOD_EXTENSION -}; - -enum HTTP_CONNECTION { - HTTP_CONNECTION_UNDEFINED = -1, - HTTP_CONNECTION_KEEP_ALIVE = 0, - HTTP_CONNECTION_CLOSE = 1 -}; - -/// Class represents general http header fields. -struct THttpBaseHeader { -public: +enum HTTP_METHOD { + HTTP_METHOD_UNDEFINED = -1, + HTTP_METHOD_OPTIONS, + HTTP_METHOD_GET, + HTTP_METHOD_HEAD, + HTTP_METHOD_POST, + HTTP_METHOD_PUT, + HTTP_METHOD_DELETE, + HTTP_METHOD_TRACE, + HTTP_METHOD_CONNECT, + HTTP_METHOD_EXTENSION +}; + +enum HTTP_CONNECTION { + HTTP_CONNECTION_UNDEFINED = -1, + HTTP_CONNECTION_KEEP_ALIVE = 0, + HTTP_CONNECTION_CLOSE = 1 +}; + +/// Class represents general http header fields. +struct THttpBaseHeader { +public: i16 error; i32 header_size; i32 entity_size; @@ -77,7 +77,7 @@ public: i8 connection_closed; TString base; -public: +public: void Init() { error = 0; header_size = 0; @@ -89,7 +89,7 @@ public: charset = -1; compression_method = HTTP_COMPRESSION_UNSET; transfer_chunked = -1; - connection_closed = HTTP_CONNECTION_UNDEFINED; + connection_closed = HTTP_CONNECTION_UNDEFINED; content_range_start = -1; content_range_end = -1; content_range_entity_length = -1; @@ -127,9 +127,9 @@ public: }; enum { HREFLANG_MAX = FETCHER_URL_MAX * 2 }; -/// Class represents Http Response Header. -struct THttpHeader: public THttpBaseHeader { -public: +/// Class represents Http Response Header. +struct THttpHeader: public THttpBaseHeader { +public: i8 accept_ranges; i8 squid_error; i8 x_robots_tag; // deprecated, use x_robots_state instead @@ -140,38 +140,38 @@ public: i64 retry_after; TString x_robots_state; // 'xxxxx' format, see `library/html/zoneconf/parsefunc.cpp` -public: - void Init() { - THttpBaseHeader::Init(); - accept_ranges = -1; +public: + void Init() { + THttpBaseHeader::Init(); + accept_ranges = -1; squid_error = 0; x_robots_tag = 0; rel_canonical.clear(); - http_status = -1; + http_status = -1; location.clear(); hreflangs[0] = 0; retry_after = DEFAULT_RETRY_AFTER; x_robots_state = "xxxxx"; - } + } - void Print() const { - THttpBaseHeader::Print(); + void Print() const { + THttpBaseHeader::Print(); printf("http_status: %" PRIi16 "\n", http_status); printf("squid_error: %" PRIi8 "\n", squid_error); printf("accept_ranges: %" PRIi8 "\n", accept_ranges); printf("location: \"%s\"\n", location.c_str()); printf("retry_after: %" PRIi64 "\n", retry_after); - } -}; - -struct THttpRequestHeader: public THttpBaseHeader { + } +}; + +struct THttpRequestHeader: public THttpBaseHeader { public: TString request_uri; - char host[HOST_MAX]; - char from[MAXWORD_LEN]; - char user_agent[MAXWORD_LEN]; + char host[HOST_MAX]; + char from[MAXWORD_LEN]; + char user_agent[MAXWORD_LEN]; char x_yandex_langregion[MAX_LANGREGION_LEN]; - char x_yandex_sourcename[MAXWORD_LEN]; + char x_yandex_sourcename[MAXWORD_LEN]; char x_yandex_requesttype[MAXWORD_LEN]; char x_yandex_fetchoptions[MAXWORD_LEN]; i8 http_method; @@ -180,78 +180,78 @@ public: i32 max_age; i64 if_modified_since; -public: - THttpRequestHeader() { - Init(); - } +public: + THttpRequestHeader() { + Init(); + } - void Init() { + void Init() { request_uri.clear(); - host[0] = 0; - from[0] = 0; - user_agent[0] = 0; + host[0] = 0; + from[0] = 0; + user_agent[0] = 0; x_yandex_langregion[0] = 0; - x_yandex_sourcename[0] = 0; + x_yandex_sourcename[0] = 0; x_yandex_requesttype[0] = 0; x_yandex_fetchoptions[0] = 0; - http_method = HTTP_METHOD_UNDEFINED; - x_yandex_request_priority = DEFAULT_REQUEST_PRIORITY; - x_yandex_response_timeout = DEFAULT_RESPONSE_TIMEOUT; - max_age = DEFAULT_MAX_AGE; - if_modified_since = DEFAULT_IF_MODIFIED_SINCE; - THttpBaseHeader::Init(); - } + http_method = HTTP_METHOD_UNDEFINED; + x_yandex_request_priority = DEFAULT_REQUEST_PRIORITY; + x_yandex_response_timeout = DEFAULT_RESPONSE_TIMEOUT; + max_age = DEFAULT_MAX_AGE; + if_modified_since = DEFAULT_IF_MODIFIED_SINCE; + THttpBaseHeader::Init(); + } - void Print() const { - THttpBaseHeader::Print(); + void Print() const { + THttpBaseHeader::Print(); printf("request_uri: \"%s\"\n", request_uri.c_str()); - printf("host: \"%s\"\n", host); - printf("from: \"%s\"\n", from); - printf("user_agent: \"%s\"\n", user_agent); + printf("host: \"%s\"\n", host); + printf("from: \"%s\"\n", from); + printf("user_agent: \"%s\"\n", user_agent); printf("http_method: %" PRIi8 "\n", http_method); printf("response_timeout: %" PRIi32 "\n", x_yandex_response_timeout); printf("max_age: %" PRIi32 "\n", max_age); printf("if_modified_since: %" PRIi64 "\n", if_modified_since); - } + } - /// It doesn't care about errors in request or headers, where - /// request_uri equals to '*'. - /// This returns copy of the string, which you have to delete. + /// It doesn't care about errors in request or headers, where + /// request_uri equals to '*'. + /// This returns copy of the string, which you have to delete. TString GetUrl() { TString url; - if (host[0] == 0 || !strcmp(host, "")) { - url = request_uri; - } else { - url = HTTP_PREFIX; - url += host; - url += request_uri; + if (host[0] == 0 || !strcmp(host, "")) { + url = request_uri; + } else { + url = HTTP_PREFIX; + url += host; + url += request_uri; } - return url; - } + return url; + } char* GetUrl(char* buffer, size_t size) { - if (host[0] == 0 || !strcmp(host, "")) { + if (host[0] == 0 || !strcmp(host, "")) { strlcpy(buffer, request_uri.c_str(), size); - } else { + } else { snprintf(buffer, size, "http://%s%s", host, request_uri.c_str()); } - return buffer; - } -}; + return buffer; + } +}; -class THttpAuthHeader: public THttpHeader { -public: - char* realm; - char* nonce; - char* opaque; +class THttpAuthHeader: public THttpHeader { +public: + char* realm; + char* nonce; + char* opaque; bool stale; int algorithm; bool qop_auth; bool use_auth; - - //we do not provide auth-int variant as too heavy - //bool qop_auth_int; - + + //we do not provide auth-int variant as too heavy + //bool qop_auth_int; + THttpAuthHeader() : realm(nullptr) , nonce(nullptr) @@ -260,28 +260,28 @@ public: , algorithm(0) , qop_auth(false) , use_auth(true) - { - THttpHeader::Init(); - } - + { + THttpHeader::Init(); + } + ~THttpAuthHeader() { - free(realm); - free(nonce); - free(opaque); - } - - void Print() { - THttpHeader::Print(); + free(realm); + free(nonce); + free(opaque); + } + + void Print() { + THttpHeader::Print(); if (use_auth) { - if (realm) - printf("realm: \"%s\"\n", realm); - if (nonce) - printf("nonce: \"%s\"\n", nonce); - if (opaque) - printf("opaque: \"%s\"\n", opaque); - printf("stale: %d\n", stale); - printf("algorithm: %d\n", algorithm); - printf("qop_auth: %d\n", qop_auth); + if (realm) + printf("realm: \"%s\"\n", realm); + if (nonce) + printf("nonce: \"%s\"\n", nonce); + if (opaque) + printf("opaque: \"%s\"\n", opaque); + printf("stale: %d\n", stale); + printf("algorithm: %d\n", algorithm); + printf("qop_auth: %d\n", qop_auth); } - } + } }; |