diff options
author | Anton Samokhvalov <pg83@yandex.ru> | 2022-02-10 16:45:15 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:15 +0300 |
commit | 72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch) | |
tree | da2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /library/cpp/http/io | |
parent | 778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff) | |
download | ydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz |
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/http/io')
-rw-r--r-- | library/cpp/http/io/chunk.cpp | 398 | ||||
-rw-r--r-- | library/cpp/http/io/chunk.h | 38 | ||||
-rw-r--r-- | library/cpp/http/io/chunk_ut.cpp | 72 | ||||
-rw-r--r-- | library/cpp/http/io/compression.cpp | 58 | ||||
-rw-r--r-- | library/cpp/http/io/compression.h | 26 | ||||
-rw-r--r-- | library/cpp/http/io/compression_ut.cpp | 20 | ||||
-rw-r--r-- | library/cpp/http/io/fuzz/main.cpp | 28 | ||||
-rw-r--r-- | library/cpp/http/io/fuzz/ya.make | 20 | ||||
-rw-r--r-- | library/cpp/http/io/headers.cpp | 106 | ||||
-rw-r--r-- | library/cpp/http/io/headers.h | 126 | ||||
-rw-r--r-- | library/cpp/http/io/headers_ut.cpp | 10 | ||||
-rw-r--r-- | library/cpp/http/io/list_codings/main.cpp | 14 | ||||
-rw-r--r-- | library/cpp/http/io/list_codings/ya.make | 24 | ||||
-rw-r--r-- | library/cpp/http/io/stream.cpp | 1302 | ||||
-rw-r--r-- | library/cpp/http/io/stream.h | 180 | ||||
-rw-r--r-- | library/cpp/http/io/stream_ut.cpp | 294 | ||||
-rw-r--r-- | library/cpp/http/io/stream_ut_medium.cpp | 94 | ||||
-rw-r--r-- | library/cpp/http/io/ut/medium/ya.make | 12 | ||||
-rw-r--r-- | library/cpp/http/io/ut/ya.make | 16 | ||||
-rw-r--r-- | library/cpp/http/io/ya.make | 24 |
20 files changed, 1431 insertions, 1431 deletions
diff --git a/library/cpp/http/io/chunk.cpp b/library/cpp/http/io/chunk.cpp index 6975d9eac1..80a423e7c9 100644 --- a/library/cpp/http/io/chunk.cpp +++ b/library/cpp/http/io/chunk.cpp @@ -1,70 +1,70 @@ -#include "chunk.h" - +#include "chunk.h" + #include "headers.h" -#include <util/string/cast.h> -#include <util/generic/utility.h> -#include <util/generic/yexception.h> - +#include <util/string/cast.h> +#include <util/generic/utility.h> +#include <util/generic/yexception.h> + static inline size_t ParseHex(const TString& s) { - if (s.empty()) { + if (s.empty()) { ythrow yexception() << "can not parse chunk length(empty string)"; - } - - size_t ret = 0; - + } + + size_t ret = 0; + for (TString::const_iterator c = s.begin(); c != s.end(); ++c) { - const char ch = *c; - - if (ch >= '0' && ch <= '9') { - ret *= 16; - ret += ch - '0'; - } else if (ch >= 'a' && ch <= 'f') { - ret *= 16; - ret += 10 + ch - 'a'; - } else if (ch >= 'A' && ch <= 'F') { - ret *= 16; - ret += 10 + ch - 'A'; - } else if (ch == ';') { - break; - } else if (isspace(ch)) { - continue; - } else { + const char ch = *c; + + if (ch >= '0' && ch <= '9') { + ret *= 16; + ret += ch - '0'; + } else if (ch >= 'a' && ch <= 'f') { + ret *= 16; + ret += 10 + ch - 'a'; + } else if (ch >= 'A' && ch <= 'F') { + ret *= 16; + ret += 10 + ch - 'A'; + } else if (ch == ';') { + break; + } else if (isspace(ch)) { + continue; + } else { ythrow yexception() << "can not parse chunk length(" << s.data() << ")"; - } - } - - return ret; -} - -static inline char* ToHex(size_t len, char* buf) { - do { - const size_t val = len % 16; - - *--buf = (val < 10) ? (val + '0') : (val - 10 + 'a'); - len /= 16; - } while (len); - - return buf; -} - -class TChunkedInput::TImpl { -public: + } + } + + return ret; +} + +static inline char* ToHex(size_t len, char* buf) { + do { + const size_t val = len % 16; + + *--buf = (val < 10) ? (val + '0') : (val - 10 + 'a'); + len /= 16; + } while (len); + + return buf; +} + +class TChunkedInput::TImpl { +public: inline TImpl(IInputStream* slave, TMaybe<THttpHeaders>* trailers) - : Slave_(slave) + : Slave_(slave) , Trailers_(trailers) - , Pending_(0) - , LastChunkReaded_(false) - { + , Pending_(0) + , LastChunkReaded_(false) + { if (Trailers_) { Trailers_->Clear(); } - } - + } + inline ~TImpl() { - } - - inline size_t Read(void* buf, size_t len) { + } + + inline size_t Read(void* buf, size_t len) { return Perform(len, [this, buf](size_t toRead) { return Slave_->Read(buf, toRead); }); } @@ -73,174 +73,174 @@ public: } private: - template <class Operation> + template <class Operation> inline size_t Perform(size_t len, const Operation& operation) { - if (!HavePendingData()) { - return 0; - } - + if (!HavePendingData()) { + return 0; + } + const size_t toProcess = Min(Pending_, len); - + if (toProcess) { const size_t processed = operation(toProcess); - + if (!processed) { - ythrow yexception() << "malformed http chunk"; - } - + ythrow yexception() << "malformed http chunk"; + } + Pending_ -= processed; - + return processed; - } - - return 0; - } - - inline bool HavePendingData() { - if (LastChunkReaded_) { - return false; - } - - if (!Pending_) { - if (!ProceedToNextChunk()) { - return false; - } - } - - return true; - } - - inline bool ProceedToNextChunk() { + } + + return 0; + } + + inline bool HavePendingData() { + if (LastChunkReaded_) { + return false; + } + + if (!Pending_) { + if (!ProceedToNextChunk()) { + return false; + } + } + + return true; + } + + inline bool ProceedToNextChunk() { TString len(Slave_->ReadLine()); - - if (len.empty()) { - /* - * skip crlf from previous chunk - */ - - len = Slave_->ReadLine(); - } - - Pending_ = ParseHex(len); - - if (Pending_) { - return true; - } - + + if (len.empty()) { + /* + * skip crlf from previous chunk + */ + + len = Slave_->ReadLine(); + } + + Pending_ = ParseHex(len); + + if (Pending_) { + return true; + } + if (Trailers_) { Trailers_->ConstructInPlace(Slave_); } - LastChunkReaded_ = true; - - return false; - } - -private: + LastChunkReaded_ = true; + + return false; + } + +private: IInputStream* Slave_; TMaybe<THttpHeaders>* Trailers_; - size_t Pending_; - bool LastChunkReaded_; -}; - + size_t Pending_; + bool LastChunkReaded_; +}; + TChunkedInput::TChunkedInput(IInputStream* slave, TMaybe<THttpHeaders>* trailers) : Impl_(new TImpl(slave, trailers)) -{ -} - +{ +} + TChunkedInput::~TChunkedInput() { -} - -size_t TChunkedInput::DoRead(void* buf, size_t len) { - return Impl_->Read(buf, len); -} - +} + +size_t TChunkedInput::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} + size_t TChunkedInput::DoSkip(size_t len) { return Impl_->Skip(len); } -class TChunkedOutput::TImpl { +class TChunkedOutput::TImpl { typedef IOutputStream::TPart TPart; - -public: + +public: inline TImpl(IOutputStream* slave) - : Slave_(slave) - { - } - + : Slave_(slave) + { + } + inline ~TImpl() { - } - - inline void Write(const void* buf, size_t len) { - const char* ptr = (const char*)buf; - - while (len) { - const size_t portion = Min<size_t>(len, 1024 * 16); - - WriteImpl(ptr, portion); - - ptr += portion; - len -= portion; - } - } - - inline void WriteImpl(const void* buf, size_t len) { - char tmp[32]; - char* e = tmp + sizeof(tmp); - char* b = ToHex(len, e); - - const TPart parts[] = { - TPart(b, e - b), - TPart::CrLf(), - TPart(buf, len), - TPart::CrLf(), - }; - - Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); - } - - inline void Flush() { - Slave_->Flush(); - } - - inline void Finish() { - Slave_->Write("0\r\n\r\n", 5); - - Flush(); - } - -private: + } + + inline void Write(const void* buf, size_t len) { + const char* ptr = (const char*)buf; + + while (len) { + const size_t portion = Min<size_t>(len, 1024 * 16); + + WriteImpl(ptr, portion); + + ptr += portion; + len -= portion; + } + } + + inline void WriteImpl(const void* buf, size_t len) { + char tmp[32]; + char* e = tmp + sizeof(tmp); + char* b = ToHex(len, e); + + const TPart parts[] = { + TPart(b, e - b), + TPart::CrLf(), + TPart(buf, len), + TPart::CrLf(), + }; + + Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); + } + + inline void Flush() { + Slave_->Flush(); + } + + inline void Finish() { + Slave_->Write("0\r\n\r\n", 5); + + Flush(); + } + +private: IOutputStream* Slave_; -}; - +}; + TChunkedOutput::TChunkedOutput(IOutputStream* slave) - : Impl_(new TImpl(slave)) -{ -} - + : Impl_(new TImpl(slave)) +{ +} + TChunkedOutput::~TChunkedOutput() { - try { - Finish(); - } catch (...) { - } -} - -void TChunkedOutput::DoWrite(const void* buf, size_t len) { - if (Impl_.Get()) { - Impl_->Write(buf, len); - } else { + try { + Finish(); + } catch (...) { + } +} + +void TChunkedOutput::DoWrite(const void* buf, size_t len) { + if (Impl_.Get()) { + Impl_->Write(buf, len); + } else { ythrow yexception() << "can not write to finished stream"; - } -} - -void TChunkedOutput::DoFlush() { - if (Impl_.Get()) { - Impl_->Flush(); - } -} - -void TChunkedOutput::DoFinish() { - if (Impl_.Get()) { - Impl_->Finish(); - Impl_.Destroy(); - } -} + } +} + +void TChunkedOutput::DoFlush() { + if (Impl_.Get()) { + Impl_->Flush(); + } +} + +void TChunkedOutput::DoFinish() { + if (Impl_.Get()) { + Impl_->Finish(); + Impl_.Destroy(); + } +} diff --git a/library/cpp/http/io/chunk.h b/library/cpp/http/io/chunk.h index 88d89fafda..a77f1b1a9d 100644 --- a/library/cpp/http/io/chunk.h +++ b/library/cpp/http/io/chunk.h @@ -1,9 +1,9 @@ #pragma once - + #include <util/stream/output.h> #include <util/generic/maybe.h> #include <util/generic/ptr.h> - + class THttpHeaders; /// @addtogroup Streams_Chunked @@ -12,36 +12,36 @@ class THttpHeaders; /// @details Последовательное чтение блоков данных. Предполагается, что /// данные записаны в виде <длина блока><блок данных>. class TChunkedInput: public IInputStream { -public: +public: /// Если передан указатель на trailers, то туда будут записаны HTTP trailer'ы (возможно пустые), /// которые идут после чанков. TChunkedInput(IInputStream* slave, TMaybe<THttpHeaders>* trailers = nullptr); ~TChunkedInput() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; size_t DoSkip(size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /// Вывод данных порциями. /// @details Вывод данных блоками в виде <длина блока><блок данных>. Если объем /// данных превышает 64K, они записываются в виде n блоков по 64K + то, что осталось. class TChunkedOutput: public IOutputStream { -public: +public: TChunkedOutput(IOutputStream* slave); ~TChunkedOutput() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; /// @} diff --git a/library/cpp/http/io/chunk_ut.cpp b/library/cpp/http/io/chunk_ut.cpp index da283f8568..82f9c90b74 100644 --- a/library/cpp/http/io/chunk_ut.cpp +++ b/library/cpp/http/io/chunk_ut.cpp @@ -1,16 +1,16 @@ -#include "chunk.h" - +#include "chunk.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/file.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <util/stream/null.h> - -#define CDATA "./chunkedio" - + +#define CDATA "./chunkedio" + Y_UNIT_TEST_SUITE(TestChunkedIO) { - static const char test_data[] = "87s6cfbsudg cuisg s igasidftasiy tfrcua6s"; - + static const char test_data[] = "87s6cfbsudg cuisg s igasidftasiy tfrcua6s"; + TString CombString(const TString& s, size_t chunkSize) { TString result; for (size_t pos = 0; pos < s.size(); pos += 2 * chunkSize) @@ -58,48 +58,48 @@ Y_UNIT_TEST_SUITE(TestChunkedIO) { } Y_UNIT_TEST(TestChunkedIo) { - TTempFile tmpFile(CDATA); + TTempFile tmpFile(CDATA); TString tmp; - - { + + { TUnbufferedFileOutput fo(CDATA); - TChunkedOutput co(&fo); + TChunkedOutput co(&fo); WriteTestData(&co, &tmp); } - + { TUnbufferedFileInput fi(CDATA); TChunkedInput ci(&fi); TString r; - + ReadInSmallChunks(&ci, &r); UNIT_ASSERT_EQUAL(r, tmp); - } - - { + } + + { TUnbufferedFileInput fi(CDATA); - TChunkedInput ci(&fi); + TChunkedInput ci(&fi); TString r; - + ReadCombed(&ci, &r, 11); - + UNIT_ASSERT_EQUAL(r, CombString(tmp, 11)); - } - } - + } + } + Y_UNIT_TEST(TestBadChunk) { - bool hasError = false; - - try { + bool hasError = false; + + try { TString badChunk = "10\r\nqwerty"; TMemoryInput mi(badChunk.data(), badChunk.size()); - TChunkedInput ci(&mi); - TransferData(&ci, &Cnull); - } catch (...) { - hasError = true; - } - - UNIT_ASSERT(hasError); - } -} + TChunkedInput ci(&mi); + TransferData(&ci, &Cnull); + } catch (...) { + hasError = true; + } + + UNIT_ASSERT(hasError); + } +} diff --git a/library/cpp/http/io/compression.cpp b/library/cpp/http/io/compression.cpp index 8fa1f62ae6..0c922ad566 100644 --- a/library/cpp/http/io/compression.cpp +++ b/library/cpp/http/io/compression.cpp @@ -1,66 +1,66 @@ #include "compression.h" - -#if defined(ENABLE_GPL) + +#if defined(ENABLE_GPL) #include <library/cpp/streams/lz/lz.h> -#endif - +#endif + #include <library/cpp/streams/brotli/brotli.h> #include <library/cpp/streams/lzma/lzma.h> #include <library/cpp/streams/bzip2/bzip2.h> - + #include <library/cpp/blockcodecs/stream.h> #include <library/cpp/blockcodecs/codecs.h> - -#include <util/stream/zlib.h> - - + +#include <util/stream/zlib.h> + + TCompressionCodecFactory::TCompressionCodecFactory() { auto gzip = [](auto s) { return MakeHolder<TZLibDecompress>(s); - }; - + }; + Add("gzip", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::GZip); }); Add("deflate", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::ZLib); }); Add("br", [](auto s) { return MakeHolder<TBrotliDecompress>(s); }, [](auto s) { return MakeHolder<TBrotliCompress>(s, 4); }); Add("x-gzip", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::GZip); }); Add("x-deflate", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::ZLib); }); - -#if defined(ENABLE_GPL) + +#if defined(ENABLE_GPL) const ui16 bs = 32 * 1024; - + Add("y-lzo", [](auto s) { return MakeHolder<TLzoDecompress>(s); }, [bs](auto s) { return MakeHolder<TLazy<TLzoCompress> >(s, bs); }); Add("y-lzf", [](auto s) { return MakeHolder<TLzfDecompress>(s); }, [bs](auto s) { return MakeHolder<TLazy<TLzfCompress> >(s, bs); }); Add("y-lzq", [](auto s) { return MakeHolder<TLzqDecompress>(s); }, [bs](auto s) { return MakeHolder<TLazy<TLzqCompress> >(s, bs); }); -#endif - +#endif + Add("y-bzip2", [](auto s) { return MakeHolder<TBZipDecompress>(s); }, [](auto s) { return MakeHolder<TBZipCompress>(s); }); Add("y-lzma", [](auto s) { return MakeHolder<TLzmaDecompress>(s); }, [](auto s) { return MakeHolder<TLzmaCompress>(s); }); - + for (auto codecName : NBlockCodecs::ListAllCodecs()) { if (codecName.StartsWith("zstd06")) { continue; - } - + } + if (codecName.StartsWith("zstd08")) { continue; - } - + } + auto codec = NBlockCodecs::Codec(codecName); - + auto enc = [codec](auto s) { return MakeHolder<NBlockCodecs::TCodedOutput>(s, codec, 32 * 1024); }; - + auto dec = [codec](auto s) { return MakeHolder<NBlockCodecs::TDecodedInput>(s, codec); - }; - + }; + Add(TString("z-") + codecName, dec, enc); - } -} - + } +} + void TCompressionCodecFactory::Add(TStringBuf name, TDecoderConstructor d, TEncoderConstructor e) { Strings_.emplace_back(name); Codecs_[Strings_.back()] = TCodec{d, e}; BestCodecs_.emplace_back(Strings_.back()); -} +} diff --git a/library/cpp/http/io/compression.h b/library/cpp/http/io/compression.h index f16c4a18eb..b566bbb796 100644 --- a/library/cpp/http/io/compression.h +++ b/library/cpp/http/io/compression.h @@ -1,17 +1,17 @@ #pragma once - + #include "stream.h" - + #include <util/generic/deque.h> #include <util/generic/hash.h> - + class TCompressionCodecFactory { -public: +public: using TDecoderConstructor = std::function<THolder<IInputStream>(IInputStream*)>; using TEncoderConstructor = std::function<THolder<IOutputStream>(IOutputStream*)>; - + TCompressionCodecFactory(); - + static inline TCompressionCodecFactory& Instance() noexcept { return *SingletonWithPriority<TCompressionCodecFactory, 0>(); } @@ -20,10 +20,10 @@ public: if (auto codec = Codecs_.FindPtr(name)) { return &codec->Decoder; } - + return nullptr; } - + inline const TEncoderConstructor* FindEncoder(TStringBuf name) const { if (auto codec = Codecs_.FindPtr(name)) { return &codec->Encoder; @@ -31,23 +31,23 @@ public: return nullptr; } - + inline TArrayRef<const TStringBuf> GetBestCodecs() const { return BestCodecs_; } -private: +private: void Add(TStringBuf name, TDecoderConstructor d, TEncoderConstructor e); - + struct TCodec { TDecoderConstructor Decoder; TEncoderConstructor Encoder; }; - + TDeque<TString> Strings_; THashMap<TStringBuf, TCodec> Codecs_; TVector<TStringBuf> BestCodecs_; -}; +}; namespace NHttp { template <typename F> diff --git a/library/cpp/http/io/compression_ut.cpp b/library/cpp/http/io/compression_ut.cpp index 2f3d131f8c..04a3f85689 100644 --- a/library/cpp/http/io/compression_ut.cpp +++ b/library/cpp/http/io/compression_ut.cpp @@ -1,34 +1,34 @@ -#include "stream.h" +#include "stream.h" #include "compression.h" - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/stream/zlib.h> + +#include <util/stream/zlib.h> #include <util/generic/hash_set.h> - + Y_UNIT_TEST_SUITE(THttpCompressionTest) { static const TString DATA = "I'm a teapot"; Y_UNIT_TEST(TestGetBestCodecs) { UNIT_ASSERT(TCompressionCodecFactory::Instance().GetBestCodecs().size() > 0); - } + } Y_UNIT_TEST(TestEncoder) { TStringStream buffer; - { + { auto encoder = TCompressionCodecFactory::Instance().FindEncoder("gzip"); UNIT_ASSERT(encoder); - + auto encodedStream = (*encoder)(&buffer); encodedStream->Write(DATA); - } + } TZLibDecompress decompressor(&buffer); UNIT_ASSERT_EQUAL(decompressor.ReadAll(), DATA); } - + Y_UNIT_TEST(TestDecoder) { TStringStream buffer; diff --git a/library/cpp/http/io/fuzz/main.cpp b/library/cpp/http/io/fuzz/main.cpp index 8ded9c7e32..60bc30848f 100644 --- a/library/cpp/http/io/fuzz/main.cpp +++ b/library/cpp/http/io/fuzz/main.cpp @@ -1,15 +1,15 @@ #include <library/cpp/http/io/stream.h> - -#include <util/generic/vector.h> -#include <util/stream/mem.h> - -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { - TMemoryInput mi(data, size); - - try { - THttpInput(&mi).ReadAll(); - } catch (...) { - } - - return 0; // Non-zero return values are reserved for future use. -} + +#include <util/generic/vector.h> +#include <util/stream/mem.h> + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + TMemoryInput mi(data, size); + + try { + THttpInput(&mi).ReadAll(); + } catch (...) { + } + + return 0; // Non-zero return values are reserved for future use. +} diff --git a/library/cpp/http/io/fuzz/ya.make b/library/cpp/http/io/fuzz/ya.make index 8b3ccb1969..2057830af1 100644 --- a/library/cpp/http/io/fuzz/ya.make +++ b/library/cpp/http/io/fuzz/ya.make @@ -1,18 +1,18 @@ FUZZ() - + OWNER( pg g:util ) - -PEERDIR( + +PEERDIR( library/cpp/http/io -) - +) + SIZE(MEDIUM) -SRCS( - main.cpp -) - -END() +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/http/io/headers.cpp b/library/cpp/http/io/headers.cpp index 4ec27a29e8..5d0d4f895d 100644 --- a/library/cpp/http/io/headers.cpp +++ b/library/cpp/http/io/headers.cpp @@ -1,64 +1,64 @@ -#include "headers.h" -#include "stream.h" - +#include "headers.h" +#include "stream.h" + #include <util/generic/strbuf.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/stream/output.h> #include <util/string/ascii.h> -#include <util/string/cast.h> -#include <util/string/strip.h> - +#include <util/string/cast.h> +#include <util/string/strip.h> + static inline TStringBuf Trim(const char* b, const char* e) noexcept { - return StripString(TStringBuf(b, e)); -} - + return StripString(TStringBuf(b, e)); +} + THttpInputHeader::THttpInputHeader(const TStringBuf header) { - size_t pos = header.find(':'); - + size_t pos = header.find(':'); + if (pos == TString::npos) { ythrow THttpParseException() << "can not parse http header(" << TString{header}.Quote() << ")"; - } - + } + Name_ = TString(header.cbegin(), header.cbegin() + pos); Value_ = ::ToString(Trim(header.cbegin() + pos + 1, header.cend())); -} - +} + THttpInputHeader::THttpInputHeader(TString name, TString value) : Name_(std::move(name)) , Value_(std::move(value)) -{ -} - +{ +} + void THttpInputHeader::OutTo(IOutputStream* stream) const { typedef IOutputStream::TPart TPart; - - const TPart parts[] = { - TPart(Name_), - TPart(": ", 2), - TPart(Value_), - TPart::CrLf(), - }; - - stream->Write(parts, sizeof(parts) / sizeof(*parts)); -} - + + const TPart parts[] = { + TPart(Name_), + TPart(": ", 2), + TPart(Value_), + TPart::CrLf(), + }; + + stream->Write(parts, sizeof(parts) / sizeof(*parts)); +} + THttpHeaders::THttpHeaders(IInputStream* stream) { TString header; TString line; - + bool rdOk = stream->ReadLine(header); while (rdOk && !header.empty()) { rdOk = stream->ReadLine(line); - + if (rdOk && ((line[0] == ' ') || (line[0] == '\t'))) { header += line; } else { AddHeader(THttpInputHeader(header)); header = line; - } - } -} - + } + } +} + bool THttpHeaders::HasHeader(const TStringBuf header) const { return FindHeader(header); } @@ -85,24 +85,24 @@ void THttpHeaders::AddOrReplaceHeader(const THttpInputHeader& header) { for (auto& hdr : Headers_) { if (AsciiCompareIgnoreCase(hdr.Name(), header.Name()) == 0) { hdr = header; - return; - } - } - - AddHeader(header); -} - + return; + } + } + + AddHeader(header); +} + void THttpHeaders::AddHeader(THttpInputHeader header) { Headers_.push_back(std::move(header)); -} - +} + void THttpHeaders::OutTo(IOutputStream* stream) const { - for (TConstIterator header = Begin(); header != End(); ++header) { - header->OutTo(stream); - } -} - -template <> + for (TConstIterator header = Begin(); header != End(); ++header) { + header->OutTo(stream); + } +} + +template <> void Out<THttpHeaders>(IOutputStream& out, const THttpHeaders& h) { - h.OutTo(&out); -} + h.OutTo(&out); +} diff --git a/library/cpp/http/io/headers.h b/library/cpp/http/io/headers.h index a71793d1c6..a6e130eaa0 100644 --- a/library/cpp/http/io/headers.h +++ b/library/cpp/http/io/headers.h @@ -1,99 +1,99 @@ #pragma once - + #include <util/generic/string.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> #include <util/generic/deque.h> #include <util/generic/vector.h> #include <util/string/cast.h> - + class IInputStream; class IOutputStream; - + /// @addtogroup Streams_HTTP /// @{ /// Объект, содержащий информацию о HTTP-заголовке. class THttpInputHeader { -public: - /// @param[in] header - строка вида 'параметр: значение'. +public: + /// @param[in] header - строка вида 'параметр: значение'. THttpInputHeader(TStringBuf header); - /// @param[in] name - имя параметра. - /// @param[in] value - значение параметра. + /// @param[in] name - имя параметра. + /// @param[in] value - значение параметра. THttpInputHeader(TString name, TString value); - - /// Возвращает имя параметра. + + /// Возвращает имя параметра. inline const TString& Name() const noexcept { - return Name_; - } - - /// Возвращает значение параметра. + return Name_; + } + + /// Возвращает значение параметра. inline const TString& Value() const noexcept { - return Value_; - } - - /// Записывает заголовок вида "имя параметра: значение\r\n" в поток. + return Value_; + } + + /// Записывает заголовок вида "имя параметра: значение\r\n" в поток. void OutTo(IOutputStream* stream) const; - - /// Возвращает строку "имя параметра: значение". + + /// Возвращает строку "имя параметра: значение". inline TString ToString() const { return Name_ + TStringBuf(": ") + Value_; - } - -private: + } + +private: TString Name_; TString Value_; -}; - +}; + /// Контейнер для хранения HTTP-заголовков -class THttpHeaders { +class THttpHeaders { using THeaders = TDeque<THttpInputHeader>; - -public: + +public: using TConstIterator = THeaders::const_iterator; - + THttpHeaders() = default; - /// Добавляет каждую строку из потока в контейнер, считая ее правильным заголовком. + /// Добавляет каждую строку из потока в контейнер, считая ее правильным заголовком. THttpHeaders(IInputStream* stream); - - /// Стандартный итератор. + + /// Стандартный итератор. inline TConstIterator Begin() const noexcept { - return Headers_.begin(); - } + return Headers_.begin(); + } inline TConstIterator begin() const noexcept { return Headers_.begin(); } - - /// Стандартный итератор. + + /// Стандартный итератор. inline TConstIterator End() const noexcept { - return Headers_.end(); - } + return Headers_.end(); + } inline TConstIterator end() const noexcept { return Headers_.end(); } - - /// Возвращает количество заголовков в контейнере. + + /// Возвращает количество заголовков в контейнере. inline size_t Count() const noexcept { - return Headers_.size(); - } - - /// Проверяет, содержит ли контейнер хотя бы один заголовок. + return Headers_.size(); + } + + /// Проверяет, содержит ли контейнер хотя бы один заголовок. inline bool Empty() const noexcept { - return Headers_.empty(); - } + return Headers_.empty(); + } - /// Добавляет заголовок в контейнер. + /// Добавляет заголовок в контейнер. void AddHeader(THttpInputHeader header); - + template <typename ValueType> void AddHeader(TString name, const ValueType& value) { AddHeader(THttpInputHeader(std::move(name), ToString(value))); } - /// Добавляет заголовок в контейнер, если тот не содержит заголовка - /// c таким же параметром. В противном случае, заменяет существующий - /// заголовок на новый. - void AddOrReplaceHeader(const THttpInputHeader& header); - + /// Добавляет заголовок в контейнер, если тот не содержит заголовка + /// c таким же параметром. В противном случае, заменяет существующий + /// заголовок на новый. + void AddOrReplaceHeader(const THttpInputHeader& header); + template <typename ValueType> void AddOrReplaceHeader(TString name, const ValueType& value) { AddOrReplaceHeader(THttpInputHeader(std::move(name), ToString(value))); @@ -109,17 +109,17 @@ public: /// Возвращает nullptr, если не нашел const THttpInputHeader* FindHeader(TStringBuf header) const; - /// Записывает все заголовки контейнера в поток. - /// @details Каждый заголовк записывается в виде "имя параметра: значение\r\n". + /// Записывает все заголовки контейнера в поток. + /// @details Каждый заголовк записывается в виде "имя параметра: значение\r\n". void OutTo(IOutputStream* stream) const; - /// Обменивает наборы заголовков двух контейнеров. + /// Обменивает наборы заголовков двух контейнеров. void Swap(THttpHeaders& headers) noexcept { - Headers_.swap(headers.Headers_); - } - -private: - THeaders Headers_; -}; - + Headers_.swap(headers.Headers_); + } + +private: + THeaders Headers_; +}; + /// @} diff --git a/library/cpp/http/io/headers_ut.cpp b/library/cpp/http/io/headers_ut.cpp index 1d23ef8fdc..b68467f786 100644 --- a/library/cpp/http/io/headers_ut.cpp +++ b/library/cpp/http/io/headers_ut.cpp @@ -13,8 +13,8 @@ namespace { THeadersExistence(const THttpHeaders& headers) { for (THttpHeaders::TConstIterator it = headers.Begin(); - it != headers.End(); - ++it) { + it != headers.End(); + ++it) { Add(it->Name(), it->Value()); } } @@ -24,7 +24,7 @@ namespace { Impl.emplace(TString(name), TString(value)); } - bool operator==(const THeadersExistence& rhs) const { + bool operator==(const THeadersExistence& rhs) const { return Impl == rhs.Impl; } @@ -34,11 +34,11 @@ namespace { }; } -bool operator==(const THeadersExistence& lhs, const THttpHeaders& rhs) { +bool operator==(const THeadersExistence& lhs, const THttpHeaders& rhs) { return lhs == THeadersExistence(rhs); } -bool operator==(const THttpHeaders& lhs, const THeadersExistence& rhs) { +bool operator==(const THttpHeaders& lhs, const THeadersExistence& rhs) { return THeadersExistence(lhs) == rhs; } diff --git a/library/cpp/http/io/list_codings/main.cpp b/library/cpp/http/io/list_codings/main.cpp index 9818d02bdf..f15b24f234 100644 --- a/library/cpp/http/io/list_codings/main.cpp +++ b/library/cpp/http/io/list_codings/main.cpp @@ -1,8 +1,8 @@ #include <library/cpp/http/io/stream.h> -#include <util/stream/output.h> - -int main() { - for (auto codec : SupportedCodings()) { - Cout << codec << Endl; - } -} +#include <util/stream/output.h> + +int main() { + for (auto codec : SupportedCodings()) { + Cout << codec << Endl; + } +} diff --git a/library/cpp/http/io/list_codings/ya.make b/library/cpp/http/io/list_codings/ya.make index e5c5fed6dc..7559ba195c 100644 --- a/library/cpp/http/io/list_codings/ya.make +++ b/library/cpp/http/io/list_codings/ya.make @@ -1,13 +1,13 @@ -PROGRAM() - -OWNER(pg) - -PEERDIR( +PROGRAM() + +OWNER(pg) + +PEERDIR( library/cpp/http/io -) - -SRCS( - main.cpp -) - -END() +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/http/io/stream.cpp b/library/cpp/http/io/stream.cpp index 6689be684f..c38faffe0b 100644 --- a/library/cpp/http/io/stream.cpp +++ b/library/cpp/http/io/stream.cpp @@ -1,156 +1,156 @@ -#include "stream.h" +#include "stream.h" #include "compression.h" -#include "chunk.h" - +#include "chunk.h" + #include <util/stream/buffered.h> -#include <util/stream/length.h> +#include <util/stream/length.h> #include <util/stream/multi.h> #include <util/stream/null.h> #include <util/stream/tee.h> - + #include <util/system/compat.h> -#include <util/system/yassert.h> - -#include <util/network/socket.h> - -#include <util/string/cast.h> +#include <util/system/yassert.h> + +#include <util/network/socket.h> + +#include <util/string/cast.h> #include <util/string/strip.h> - + #include <util/generic/string.h> -#include <util/generic/utility.h> -#include <util/generic/hash_set.h> -#include <util/generic/yexception.h> - -#define HEADERCMP(header, str) \ - case sizeof(str) - 1: \ +#include <util/generic/utility.h> +#include <util/generic/hash_set.h> +#include <util/generic/yexception.h> + +#define HEADERCMP(header, str) \ + case sizeof(str) - 1: \ if (!stricmp((header).Name().data(), str)) -namespace { +namespace { inline size_t SuggestBufferSize() { - return 8192; - } - + return 8192; + } + inline TStringBuf Trim(const char* b, const char* e) noexcept { - return StripString(TStringBuf(b, e)); - } - + return StripString(TStringBuf(b, e)); + } + inline TStringBuf RmSemiColon(const TStringBuf& s) { - return s.Before(';'); - } - - template <class T, size_t N> - class TStreams: private TNonCopyable { - struct TDelete { - inline void operator()(T* t) noexcept { - delete t; - } - }; - - typedef T* TPtr; - - public: - inline TStreams() noexcept - : Beg_(T_ + N) - { - } - - inline ~TStreams() { - TDelete f; - - ForEach(f); - } - - template <class S> - inline S* Add(S* t) noexcept { - return (S*)AddImpl((T*)t); - } - - template <class Functor> - inline void ForEach(Functor& f) { - const TPtr* end = T_ + N; - - for (TPtr* cur = Beg_; cur != end; ++cur) { - f(*cur); - } - } - - TPtr Top() { - const TPtr* end = T_ + N; - return end == Beg_ ? nullptr : *Beg_; - } - - private: - inline T* AddImpl(T* t) noexcept { - Y_ASSERT(Beg_ > T_); - - return (*--Beg_ = t); - } - - private: - TPtr T_[N]; - TPtr* Beg_; - }; - - template <class TStream> - class TLazy: public IOutputStream { - public: - TLazy(IOutputStream* out, ui16 bs) - : Output_(out) - , BlockSize_(bs) - { - } - - void DoWrite(const void* buf, size_t len) override { - ConstructSlave(); - Slave_->Write(buf, len); - } - - void DoFlush() override { - ConstructSlave(); - Slave_->Flush(); - } - - void DoFinish() override { - ConstructSlave(); - Slave_->Finish(); - } - - private: - inline void ConstructSlave() { - if (!Slave_) { - Slave_.Reset(new TStream(Output_, BlockSize_)); - } - } - - private: - IOutputStream* Output_; - ui16 BlockSize_; - THolder<IOutputStream> Slave_; - }; -} - -class THttpInput::TImpl { + return s.Before(';'); + } + + template <class T, size_t N> + class TStreams: private TNonCopyable { + struct TDelete { + inline void operator()(T* t) noexcept { + delete t; + } + }; + + typedef T* TPtr; + + public: + inline TStreams() noexcept + : Beg_(T_ + N) + { + } + + inline ~TStreams() { + TDelete f; + + ForEach(f); + } + + template <class S> + inline S* Add(S* t) noexcept { + return (S*)AddImpl((T*)t); + } + + template <class Functor> + inline void ForEach(Functor& f) { + const TPtr* end = T_ + N; + + for (TPtr* cur = Beg_; cur != end; ++cur) { + f(*cur); + } + } + + TPtr Top() { + const TPtr* end = T_ + N; + return end == Beg_ ? nullptr : *Beg_; + } + + private: + inline T* AddImpl(T* t) noexcept { + Y_ASSERT(Beg_ > T_); + + return (*--Beg_ = t); + } + + private: + TPtr T_[N]; + TPtr* Beg_; + }; + + template <class TStream> + class TLazy: public IOutputStream { + public: + TLazy(IOutputStream* out, ui16 bs) + : Output_(out) + , BlockSize_(bs) + { + } + + void DoWrite(const void* buf, size_t len) override { + ConstructSlave(); + Slave_->Write(buf, len); + } + + void DoFlush() override { + ConstructSlave(); + Slave_->Flush(); + } + + void DoFinish() override { + ConstructSlave(); + Slave_->Finish(); + } + + private: + inline void ConstructSlave() { + if (!Slave_) { + Slave_.Reset(new TStream(Output_, BlockSize_)); + } + } + + private: + IOutputStream* Output_; + ui16 BlockSize_; + THolder<IOutputStream> Slave_; + }; +} + +class THttpInput::TImpl { typedef THashSet<TString> TAcceptCodings; - -public: + +public: inline TImpl(IInputStream* slave) - : Slave_(slave) - , Buffered_(Slave_, SuggestBufferSize()) + : Slave_(slave) + , Buffered_(Slave_, SuggestBufferSize()) , ChunkedInput_(nullptr) , Input_(nullptr) , FirstLine_(ReadFirstLine(Buffered_)) - , Headers_(&Buffered_) - , KeepAlive_(false) - , HasContentLength_(false) - , ContentLength_(0) - , ContentEncoded_(false) + , Headers_(&Buffered_) + , KeepAlive_(false) + , HasContentLength_(false) + , ContentLength_(0) + , ContentEncoded_(false) , Expect100Continue_(false) - { - BuildInputChain(); + { + BuildInputChain(); Y_ASSERT(Input_); - } - + } + static TString ReadFirstLine(TBufferedInput& in) { TString s; Y_ENSURE_EX(in.ReadLine(s), THttpReadException() << "Failed to get first line"); @@ -158,48 +158,48 @@ public: } inline ~TImpl() { - } - - inline size_t Read(void* buf, size_t len) { + } + + inline size_t Read(void* buf, size_t len) { return Perform(len, [this, buf](size_t toRead) { return Input_->Read(buf, toRead); }); - } + } inline size_t Skip(size_t len) { return Perform(len, [this](size_t toSkip) { return Input_->Skip(toSkip); }); - } - + } + inline const TString& FirstLine() const noexcept { - return FirstLine_; - } - + return FirstLine_; + } + inline const THttpHeaders& Headers() const noexcept { - return Headers_; - } - + return Headers_; + } + inline const TMaybe<THttpHeaders>& Trailers() const noexcept { return Trailers_; } inline bool IsKeepAlive() const noexcept { - return KeepAlive_; - } - + return KeepAlive_; + } + inline bool AcceptEncoding(const TString& s) const { - return Codings_.find(to_lower(s)) != Codings_.end(); - } - + return Codings_.find(to_lower(s)) != Codings_.end(); + } + inline bool GetContentLength(ui64& value) const noexcept { - if (HasContentLength_) { - value = ContentLength_; - return true; + if (HasContentLength_) { + value = ContentLength_; + return true; } - return false; - } + return false; + } inline bool ContentEncoded() const noexcept { - return ContentEncoded_; - } - + return ContentEncoded_; + } + inline bool HasContent() const noexcept { return HasContentLength_ || ChunkedInput_; } @@ -208,8 +208,8 @@ public: return Expect100Continue_; } -private: - template <class Operation> +private: + template <class Operation> inline size_t Perform(size_t len, const Operation& operation) { size_t processed = operation(len); if (processed == 0 && len > 0) { @@ -227,52 +227,52 @@ private: return processed; } - struct TParsedHeaders { + struct TParsedHeaders { bool Chunked = false; bool KeepAlive = false; - TStringBuf LZipped; - }; - - struct TTrEnc { - inline void operator()(const TStringBuf& s) { + TStringBuf LZipped; + }; + + struct TTrEnc { + inline void operator()(const TStringBuf& s) { if (s == TStringBuf("chunked")) { p->Chunked = true; - } - } - - TParsedHeaders* p; - }; - - struct TAccCoding { - inline void operator()(const TStringBuf& s) { - c->insert(ToString(s)); - } - - TAcceptCodings* c; - }; - - template <class Functor> + } + } + + TParsedHeaders* p; + }; + + struct TAccCoding { + inline void operator()(const TStringBuf& s) { + c->insert(ToString(s)); + } + + TAcceptCodings* c; + }; + + template <class Functor> inline void ForEach(TString in, Functor& f) { - in.to_lower(); - - const char* b = in.begin(); - const char* c = b; - const char* e = in.end(); - - while (c != e) { - if (*c == ',') { - f(RmSemiColon(Trim(b, c))); - b = c + 1; - } - - ++c; - } - - if (b != c) { - f(RmSemiColon(Trim(b, c))); - } - } - + in.to_lower(); + + const char* b = in.begin(); + const char* c = b; + const char* e = in.end(); + + while (c != e) { + if (*c == ',') { + f(RmSemiColon(Trim(b, c))); + b = c + 1; + } + + ++c; + } + + if (b != c) { + f(RmSemiColon(Trim(b, c))); + } + } + inline bool IsRequest() const { return strnicmp(FirstLine().data(), "get", 3) == 0 || strnicmp(FirstLine().data(), "post", 4) == 0 || @@ -282,47 +282,47 @@ private: strnicmp(FirstLine().data(), "delete", 6) == 0; } - inline void BuildInputChain() { - TParsedHeaders p; + inline void BuildInputChain() { + TParsedHeaders p; - size_t pos = FirstLine_.rfind(' '); + size_t pos = FirstLine_.rfind(' '); // In HTTP/1.1 Keep-Alive is turned on by default if (pos != TString::npos && strcmp(FirstLine_.c_str() + pos + 1, "HTTP/1.1") == 0) { - p.KeepAlive = true; //request + p.KeepAlive = true; //request } else if (strnicmp(FirstLine_.data(), "HTTP/1.1", 8) == 0) { - p.KeepAlive = true; //reply - } - - for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { - const THttpInputHeader& header = *h; + p.KeepAlive = true; //reply + } + + for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { + const THttpInputHeader& header = *h; switch (header.Name().size()) { - HEADERCMP(header, "transfer-encoding") { - TTrEnc f = {&p}; - ForEach(header.Value(), f); - } - break; - HEADERCMP(header, "content-encoding") { - p.LZipped = header.Value(); - } - break; - HEADERCMP(header, "accept-encoding") { - TAccCoding f = {&Codings_}; - ForEach(header.Value(), f); - } - break; - HEADERCMP(header, "content-length") { - HasContentLength_ = true; - ContentLength_ = FromString(header.Value()); - } - break; - HEADERCMP(header, "connection") { - // accept header "Connection: Keep-Alive, TE" + HEADERCMP(header, "transfer-encoding") { + TTrEnc f = {&p}; + ForEach(header.Value(), f); + } + break; + HEADERCMP(header, "content-encoding") { + p.LZipped = header.Value(); + } + break; + HEADERCMP(header, "accept-encoding") { + TAccCoding f = {&Codings_}; + ForEach(header.Value(), f); + } + break; + HEADERCMP(header, "content-length") { + HasContentLength_ = true; + ContentLength_ = FromString(header.Value()); + } + break; + HEADERCMP(header, "connection") { + // accept header "Connection: Keep-Alive, TE" if (strnicmp(header.Value().data(), "keep-alive", 10) == 0) { p.KeepAlive = true; } else if (stricmp(header.Value().data(), "close") == 0) { p.KeepAlive = false; } - } + } [[fallthrough]]; HEADERCMP(header, "expect") { auto findContinue = [&](const TStringBuf& s) { @@ -332,101 +332,101 @@ private: }; ForEach(header.Value(), findContinue); } - break; - } - } - + break; + } + } + if (p.Chunked) { ChunkedInput_ = Streams_.Add(new TChunkedInput(&Buffered_, &Trailers_)); Input_ = ChunkedInput_; - } else { + } else { // disable buffering - Buffered_.Reset(&Cnull); - Input_ = Streams_.Add(new TMultiInput(&Buffered_, Slave_)); - + Buffered_.Reset(&Cnull); + Input_ = Streams_.Add(new TMultiInput(&Buffered_, Slave_)); + if (IsRequest() || HasContentLength_) { - /* - * TODO - we have other cases - */ - Input_ = Streams_.Add(new TLengthLimitedInput(Input_, ContentLength_)); - } - } - + /* + * TODO - we have other cases + */ + Input_ = Streams_.Add(new TLengthLimitedInput(Input_, ContentLength_)); + } + } + if (auto decoder = TCompressionCodecFactory::Instance().FindDecoder(p.LZipped)) { - ContentEncoded_ = true; + ContentEncoded_ = true; Input_ = Streams_.Add((*decoder)(Input_).Release()); - } - + } + KeepAlive_ = p.KeepAlive; - } - -private: + } + +private: IInputStream* Slave_; - - /* - * input helpers - */ - TBufferedInput Buffered_; + + /* + * input helpers + */ + TBufferedInput Buffered_; TStreams<IInputStream, 8> Streams_; IInputStream* ChunkedInput_; - - /* - * final input stream - */ + + /* + * final input stream + */ IInputStream* Input_; - + TString FirstLine_; - THttpHeaders Headers_; + THttpHeaders Headers_; TMaybe<THttpHeaders> Trailers_; - bool KeepAlive_; + bool KeepAlive_; + + TAcceptCodings Codings_; - TAcceptCodings Codings_; + bool HasContentLength_; + ui64 ContentLength_; - bool HasContentLength_; - ui64 ContentLength_; - - bool ContentEncoded_; + bool ContentEncoded_; bool Expect100Continue_; -}; - +}; + THttpInput::THttpInput(IInputStream* slave) - : Impl_(new TImpl(slave)) -{ -} - + : Impl_(new TImpl(slave)) +{ +} + THttpInput::THttpInput(THttpInput&& httpInput) = default; THttpInput::~THttpInput() { -} - -size_t THttpInput::DoRead(void* buf, size_t len) { - return Impl_->Read(buf, len); -} - +} + +size_t THttpInput::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} + size_t THttpInput::DoSkip(size_t len) { return Impl_->Skip(len); } const THttpHeaders& THttpInput::Headers() const noexcept { - return Impl_->Headers(); -} - + return Impl_->Headers(); +} + const TMaybe<THttpHeaders>& THttpInput::Trailers() const noexcept { return Impl_->Trailers(); } const TString& THttpInput::FirstLine() const noexcept { - return Impl_->FirstLine(); -} - + return Impl_->FirstLine(); +} + bool THttpInput::IsKeepAlive() const noexcept { - return Impl_->IsKeepAlive(); -} - + return Impl_->IsKeepAlive(); +} + bool THttpInput::AcceptEncoding(const TString& coding) const { - return Impl_->AcceptEncoding(coding); -} - + return Impl_->AcceptEncoding(coding); +} + TString THttpInput::BestCompressionScheme(TArrayRef<const TStringBuf> codings) const { return NHttp::ChooseBestCompressionScheme( [this](const TString& coding) { @@ -434,8 +434,8 @@ TString THttpInput::BestCompressionScheme(TArrayRef<const TStringBuf> codings) c }, codings ); -} - +} + TString THttpInput::BestCompressionScheme() const { return BestCompressionScheme(TCompressionCodecFactory::Instance().GetBestCodecs()); } @@ -456,141 +456,141 @@ bool THttpInput::HasExpect100Continue() const noexcept { return Impl_->HasExpect100Continue(); } -class THttpOutput::TImpl { +class THttpOutput::TImpl { class TSizeCalculator: public IOutputStream { - public: + public: inline TSizeCalculator() noexcept { - } - + } + ~TSizeCalculator() override { - } - + } + void DoWrite(const void* /*buf*/, size_t len) override { - Length_ += len; - } - + Length_ += len; + } + inline size_t Length() const noexcept { - return Length_; - } - - private: + return Length_; + } + + private: size_t Length_ = 0; - }; - - enum TState { - Begin = 0, - FirstLineSent = 1, - HeadersSent = 2 - }; - - struct TFlush { + }; + + enum TState { + Begin = 0, + FirstLineSent = 1, + HeadersSent = 2 + }; + + struct TFlush { inline void operator()(IOutputStream* s) { - s->Flush(); - } - }; - - struct TFinish { + s->Flush(); + } + }; + + struct TFinish { inline void operator()(IOutputStream* s) { - s->Finish(); - } - }; - -public: + s->Finish(); + } + }; + +public: inline TImpl(IOutputStream* slave, THttpInput* request) - : Slave_(slave) - , State_(Begin) - , Output_(Slave_) - , Request_(request) - , Version_(1100) - , KeepAliveEnabled_(false) + : Slave_(slave) + , State_(Begin) + , Output_(Slave_) + , Request_(request) + , Version_(1100) + , KeepAliveEnabled_(false) , BodyEncodingEnabled_(true) , CompressionHeaderEnabled_(true) - , Finished_(false) - { - } - + , Finished_(false) + { + } + inline ~TImpl() { - } - + } + inline void SendContinue() { Output_->Write("HTTP/1.1 100 Continue\r\n\r\n"); Output_->Flush(); } - inline void Write(const void* buf, size_t len) { - if (Finished_) { - ythrow THttpException() << "can not write to finished stream"; - } - - if (State_ == HeadersSent) { - Output_->Write(buf, len); - - return; - } - - const char* b = (const char*)buf; - const char* e = b + len; - const char* c = b; - - while (c != e) { - if (*c == '\n') { - Line_.append(b, c); - + inline void Write(const void* buf, size_t len) { + if (Finished_) { + ythrow THttpException() << "can not write to finished stream"; + } + + if (State_ == HeadersSent) { + Output_->Write(buf, len); + + return; + } + + const char* b = (const char*)buf; + const char* e = b + len; + const char* c = b; + + while (c != e) { + if (*c == '\n') { + Line_.append(b, c); + if (!Line_.empty() && Line_.back() == '\r') { - Line_.pop_back(); - } - - b = c + 1; - - Process(Line_); - - if (State_ == HeadersSent) { - Output_->Write(b, e - b); - - return; - } - - Line_.clear(); - } - - ++c; - } - - if (b != c) { - Line_.append(b, c); - } - } - - inline void Flush() { - TFlush f; - Streams_.ForEach(f); + Line_.pop_back(); + } + + b = c + 1; + + Process(Line_); + + if (State_ == HeadersSent) { + Output_->Write(b, e - b); + + return; + } + + Line_.clear(); + } + + ++c; + } + + if (b != c) { + Line_.append(b, c); + } + } + + inline void Flush() { + TFlush f; + Streams_.ForEach(f); Slave_->Flush(); // see SEARCH-1030 - } - - inline void Finish() { - if (Finished_) { - return; - } - - TFinish f; - Streams_.ForEach(f); + } + + inline void Finish() { + if (Finished_) { + return; + } + + TFinish f; + Streams_.ForEach(f); Slave_->Finish(); // see SEARCH-1030 - - Finished_ = true; - } - + + Finished_ = true; + } + inline const THttpHeaders& SentHeaders() const noexcept { - return Headers_; - } - + return Headers_; + } + inline void EnableCompression(TArrayRef<const TStringBuf> schemas) { - ComprSchemas_ = schemas; - } - - inline void EnableKeepAlive(bool enable) { - KeepAliveEnabled_ = enable; - } - + ComprSchemas_ = schemas; + } + + inline void EnableKeepAlive(bool enable) { + KeepAliveEnabled_ = enable; + } + inline void EnableBodyEncoding(bool enable) { BodyEncodingEnabled_ = enable; } @@ -601,12 +601,12 @@ public: inline bool IsCompressionEnabled() const noexcept { return !ComprSchemas_.empty(); - } - + } + inline bool IsKeepAliveEnabled() const noexcept { - return KeepAliveEnabled_; - } - + return KeepAliveEnabled_; + } + inline bool IsBodyEncodingEnabled() const noexcept { return BodyEncodingEnabled_; } @@ -616,9 +616,9 @@ public: } inline bool CanBeKeepAlive() const noexcept { - return SupportChunkedTransfer() && IsKeepAliveEnabled() && (Request_ ? Request_->IsKeepAlive() : true); - } - + return SupportChunkedTransfer() && IsKeepAliveEnabled() && (Request_ ? Request_->IsKeepAlive() : true); + } + inline const TString& FirstLine() const noexcept { return FirstLine_; } @@ -627,18 +627,18 @@ public: return SizeCalculator_.Length(); } -private: +private: static inline bool IsResponse(const TString& s) noexcept { return strnicmp(s.data(), "HTTP/", 5) == 0; - } - + } + static inline bool IsRequest(const TString& s) noexcept { - return !IsResponse(s); - } - + return !IsResponse(s); + } + inline bool IsHttpRequest() const noexcept { - return IsRequest(FirstLine_); - } + return IsRequest(FirstLine_); + } inline bool HasResponseBody() const noexcept { if (IsHttpResponse()) { @@ -652,169 +652,169 @@ private: } inline bool IsHttpResponse() const noexcept { - return IsResponse(FirstLine_); - } - + return IsResponse(FirstLine_); + } + inline bool HasRequestBody() const noexcept { return strnicmp(FirstLine_.data(), "POST", 4) == 0 || strnicmp(FirstLine_.data(), "PATCH", 5) == 0 || strnicmp(FirstLine_.data(), "PUT", 3) == 0; - } + } static inline size_t ParseHttpVersion(const TString& s) { - if (s.empty()) { - ythrow THttpParseException() << "malformed http stream"; + if (s.empty()) { + ythrow THttpParseException() << "malformed http stream"; } - - size_t parsed_version = 0; - - if (IsResponse(s)) { + + size_t parsed_version = 0; + + if (IsResponse(s)) { const char* b = s.data() + 5; - - while (*b && *b != ' ') { - if (*b != '.') { - parsed_version *= 10; - parsed_version += (*b - '0'); - } - - ++b; - } - } else { - /* - * s not empty here - */ - const char* e = s.end() - 1; - const char* b = s.begin(); - size_t mult = 1; - - while (e != b && *e != '/') { - if (*e != '.') { - parsed_version += (*e - '0') * mult; - mult *= 10; - } - - --e; - } - } - - return parsed_version * 100; - } - - inline void ParseHttpVersion() { - size_t parsed_version = ParseHttpVersion(FirstLine_); - - if (Request_) { - parsed_version = Min(parsed_version, ParseHttpVersion(Request_->FirstLine())); - } - - Version_ = parsed_version; - } - + + while (*b && *b != ' ') { + if (*b != '.') { + parsed_version *= 10; + parsed_version += (*b - '0'); + } + + ++b; + } + } else { + /* + * s not empty here + */ + const char* e = s.end() - 1; + const char* b = s.begin(); + size_t mult = 1; + + while (e != b && *e != '/') { + if (*e != '.') { + parsed_version += (*e - '0') * mult; + mult *= 10; + } + + --e; + } + } + + return parsed_version * 100; + } + + inline void ParseHttpVersion() { + size_t parsed_version = ParseHttpVersion(FirstLine_); + + if (Request_) { + parsed_version = Min(parsed_version, ParseHttpVersion(Request_->FirstLine())); + } + + Version_ = parsed_version; + } + inline void Process(const TString& s) { Y_ASSERT(State_ != HeadersSent); - - if (State_ == Begin) { - FirstLine_ = s; - ParseHttpVersion(); - State_ = FirstLineSent; - } else { - if (s.empty()) { - BuildOutputStream(); - WriteCached(); - State_ = HeadersSent; - } else { + + if (State_ == Begin) { + FirstLine_ = s; + ParseHttpVersion(); + State_ = FirstLineSent; + } else { + if (s.empty()) { + BuildOutputStream(); + WriteCached(); + State_ = HeadersSent; + } else { AddHeader(THttpInputHeader(s)); - } - } - } - + } + } + } + inline void WriteCachedImpl(IOutputStream* s) const { s->Write(FirstLine_.data(), FirstLine_.size()); - s->Write("\r\n", 2); - Headers_.OutTo(s); - s->Write("\r\n", 2); - s->Finish(); - } - - inline void WriteCached() { - size_t buflen = 0; - - { - TSizeCalculator out; - - WriteCachedImpl(&out); - buflen = out.Length(); - } - - { - TBufferedOutput out(Slave_, buflen); - - WriteCachedImpl(&out); - } - - if (IsHttpRequest() && !HasRequestBody()) { - /* - * if this is http request, then send it now - */ - - Slave_->Flush(); - } - } - + s->Write("\r\n", 2); + Headers_.OutTo(s); + s->Write("\r\n", 2); + s->Finish(); + } + + inline void WriteCached() { + size_t buflen = 0; + + { + TSizeCalculator out; + + WriteCachedImpl(&out); + buflen = out.Length(); + } + + { + TBufferedOutput out(Slave_, buflen); + + WriteCachedImpl(&out); + } + + if (IsHttpRequest() && !HasRequestBody()) { + /* + * if this is http request, then send it now + */ + + Slave_->Flush(); + } + } + inline bool SupportChunkedTransfer() const noexcept { - return Version_ >= 1100; - } - - inline void BuildOutputStream() { - if (CanBeKeepAlive()) { - AddOrReplaceHeader(THttpInputHeader("Connection", "Keep-Alive")); - } else { - AddOrReplaceHeader(THttpInputHeader("Connection", "Close")); - } - - if (IsHttpResponse()) { + return Version_ >= 1100; + } + + inline void BuildOutputStream() { + if (CanBeKeepAlive()) { + AddOrReplaceHeader(THttpInputHeader("Connection", "Keep-Alive")); + } else { + AddOrReplaceHeader(THttpInputHeader("Connection", "Close")); + } + + if (IsHttpResponse()) { if (Request_ && IsCompressionEnabled() && HasResponseBody()) { TString scheme = Request_->BestCompressionScheme(ComprSchemas_); - if (scheme != "identity") { - AddOrReplaceHeader(THttpInputHeader("Content-Encoding", scheme)); + if (scheme != "identity") { + AddOrReplaceHeader(THttpInputHeader("Content-Encoding", scheme)); RemoveHeader("Content-Length"); - } - } - - RebuildStream(); - } else { - if (IsCompressionEnabled()) { - AddOrReplaceHeader(THttpInputHeader("Accept-Encoding", BuildAcceptEncoding())); - } - if (HasRequestBody()) { - RebuildStream(); - } - } - } - + } + } + + RebuildStream(); + } else { + if (IsCompressionEnabled()) { + AddOrReplaceHeader(THttpInputHeader("Accept-Encoding", BuildAcceptEncoding())); + } + if (HasRequestBody()) { + RebuildStream(); + } + } + } + inline TString BuildAcceptEncoding() const { TString ret; - + for (const auto& coding : ComprSchemas_) { - if (ret) { - ret += ", "; - } - + if (ret) { + ret += ", "; + } + ret += coding; - } - - return ret; - } + } + + return ret; + } - inline void RebuildStream() { + inline void RebuildStream() { bool keepAlive = false; const TCompressionCodecFactory::TEncoderConstructor* encoder = nullptr; - bool chunked = false; + bool chunked = false; bool haveContentLength = false; - - for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { - const THttpInputHeader& header = *h; + + for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { + const THttpInputHeader& header = *h; const TString hl = to_lower(header.Name()); - + if (hl == TStringBuf("connection")) { keepAlive = to_lower(header.Value()) == TStringBuf("keep-alive"); } else if (IsCompressionHeaderEnabled() && hl == TStringBuf("content-encoding")) { @@ -823,109 +823,109 @@ private: chunked = to_lower(header.Value()) == TStringBuf("chunked"); } else if (hl == TStringBuf("content-length")) { haveContentLength = true; - } - } - + } + } + if (!haveContentLength && !chunked && (IsHttpRequest() || HasResponseBody()) && SupportChunkedTransfer() && (keepAlive || encoder || IsHttpRequest())) { - AddHeader(THttpInputHeader("Transfer-Encoding", "chunked")); - chunked = true; - } - + AddHeader(THttpInputHeader("Transfer-Encoding", "chunked")); + chunked = true; + } + if (IsBodyEncodingEnabled() && chunked) { - Output_ = Streams_.Add(new TChunkedOutput(Output_)); - } - + Output_ = Streams_.Add(new TChunkedOutput(Output_)); + } + Output_ = Streams_.Add(new TTeeOutput(Output_, &SizeCalculator_)); if (IsBodyEncodingEnabled() && encoder) { Output_ = Streams_.Add((*encoder)(Output_).Release()); - } - } - - inline void AddHeader(const THttpInputHeader& hdr) { - Headers_.AddHeader(hdr); - } - - inline void AddOrReplaceHeader(const THttpInputHeader& hdr) { - Headers_.AddOrReplaceHeader(hdr); - } - + } + } + + inline void AddHeader(const THttpInputHeader& hdr) { + Headers_.AddHeader(hdr); + } + + inline void AddOrReplaceHeader(const THttpInputHeader& hdr) { + Headers_.AddOrReplaceHeader(hdr); + } + inline void RemoveHeader(const TString& hdr) { Headers_.RemoveHeader(hdr); } -private: +private: IOutputStream* Slave_; - TState State_; + TState State_; IOutputStream* Output_; TStreams<IOutputStream, 8> Streams_; TString Line_; TString FirstLine_; - THttpHeaders Headers_; - THttpInput* Request_; - size_t Version_; - + THttpHeaders Headers_; + THttpInput* Request_; + size_t Version_; + TArrayRef<const TStringBuf> ComprSchemas_; - - bool KeepAliveEnabled_; + + bool KeepAliveEnabled_; bool BodyEncodingEnabled_; bool CompressionHeaderEnabled_; - bool Finished_; + bool Finished_; TSizeCalculator SizeCalculator_; -}; - +}; + THttpOutput::THttpOutput(IOutputStream* slave) : Impl_(new TImpl(slave, nullptr)) -{ -} - +{ +} + THttpOutput::THttpOutput(IOutputStream* slave, THttpInput* request) - : Impl_(new TImpl(slave, request)) -{ -} - + : Impl_(new TImpl(slave, request)) +{ +} + THttpOutput::~THttpOutput() { - try { - Finish(); - } catch (...) { - } -} - -void THttpOutput::DoWrite(const void* buf, size_t len) { - Impl_->Write(buf, len); -} - -void THttpOutput::DoFlush() { - Impl_->Flush(); -} - -void THttpOutput::DoFinish() { - Impl_->Finish(); -} - + try { + Finish(); + } catch (...) { + } +} + +void THttpOutput::DoWrite(const void* buf, size_t len) { + Impl_->Write(buf, len); +} + +void THttpOutput::DoFlush() { + Impl_->Flush(); +} + +void THttpOutput::DoFinish() { + Impl_->Finish(); +} + const THttpHeaders& THttpOutput::SentHeaders() const noexcept { return Impl_->SentHeaders(); -} - -void THttpOutput::EnableCompression(bool enable) { - if (enable) { +} + +void THttpOutput::EnableCompression(bool enable) { + if (enable) { EnableCompression(TCompressionCodecFactory::Instance().GetBestCodecs()); - } else { + } else { TArrayRef<TStringBuf> codings; EnableCompression(codings); - } -} - + } +} + void THttpOutput::EnableCompression(TArrayRef<const TStringBuf> schemas) { Impl_->EnableCompression(schemas); -} - -void THttpOutput::EnableKeepAlive(bool enable) { - Impl_->EnableKeepAlive(enable); -} - +} + +void THttpOutput::EnableKeepAlive(bool enable) { + Impl_->EnableKeepAlive(enable); +} + void THttpOutput::EnableBodyEncoding(bool enable) { Impl_->EnableBodyEncoding(enable); } @@ -935,25 +935,25 @@ void THttpOutput::EnableCompressionHeader(bool enable) { } bool THttpOutput::IsKeepAliveEnabled() const noexcept { - return Impl_->IsKeepAliveEnabled(); -} - + return Impl_->IsKeepAliveEnabled(); +} + bool THttpOutput::IsBodyEncodingEnabled() const noexcept { return Impl_->IsBodyEncodingEnabled(); } bool THttpOutput::IsCompressionEnabled() const noexcept { - return Impl_->IsCompressionEnabled(); -} - + return Impl_->IsCompressionEnabled(); +} + bool THttpOutput::IsCompressionHeaderEnabled() const noexcept { return Impl_->IsCompressionHeaderEnabled(); } bool THttpOutput::CanBeKeepAlive() const noexcept { - return Impl_->CanBeKeepAlive(); -} - + return Impl_->CanBeKeepAlive(); +} + void THttpOutput::SendContinue() { Impl_->SendContinue(); } @@ -966,13 +966,13 @@ size_t THttpOutput::SentSize() const noexcept { return Impl_->SentSize(); } -unsigned ParseHttpRetCode(const TStringBuf& ret) { - const TStringBuf code = StripString(StripString(ret.After(' ')).Before(' ')); - +unsigned ParseHttpRetCode(const TStringBuf& ret) { + const TStringBuf code = StripString(StripString(ret.After(' ')).Before(' ')); + return FromString<unsigned>(code.data(), code.size()); -} +} -void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent, const TStringBuf& from) { +void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent, const TStringBuf& from) { TSocketOutput so(s); THttpOutput output(&so); @@ -995,11 +995,11 @@ void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf IOutputStream::TPart::CrLf(), IOutputStream::TPart::CrLf(), }; - + output.Write(parts, sizeof(parts) / sizeof(*parts)); output.Finish(); } - + TArrayRef<const TStringBuf> SupportedCodings() { return TCompressionCodecFactory::Instance().GetBestCodecs(); -} +} diff --git a/library/cpp/http/io/stream.h b/library/cpp/http/io/stream.h index 78ca4fc814..ab69a8abab 100644 --- a/library/cpp/http/io/stream.h +++ b/library/cpp/http/io/stream.h @@ -1,40 +1,40 @@ #pragma once - -#include "headers.h" - + +#include "headers.h" + #include <util/stream/output.h> #include <util/generic/maybe.h> #include <util/generic/ptr.h> #include <util/generic/string.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> #include <util/generic/yexception.h> -#include <util/generic/array_ref.h> - -class TSocket; - -struct THttpException: public yexception { -}; - -struct THttpParseException: public THttpException { -}; - +#include <util/generic/array_ref.h> + +class TSocket; + +struct THttpException: public yexception { +}; + +struct THttpParseException: public THttpException { +}; + struct THttpReadException: public THttpException { }; /// Чтение ответа HTTP-сервера. class THttpInput: public IInputStream { -public: +public: THttpInput(IInputStream* slave); THttpInput(THttpInput&& httpInput); ~THttpInput() override; - - /* - * parsed http headers - */ - /// Возвращает контейнер с заголовками ответа HTTP-сервера. + + /* + * parsed http headers + */ + /// Возвращает контейнер с заголовками ответа HTTP-сервера. const THttpHeaders& Headers() const noexcept; - - /* + + /* * parsed http trailers */ /// Возвращает контейнер (возможно пустой) с trailer'ами ответа HTTP-сервера. @@ -44,78 +44,78 @@ public: const TMaybe<THttpHeaders>& Trailers() const noexcept; /* - * first line - response or request - */ - /// Возвращает первую строку ответа HTTP-сервера. - /// @details Первая строка HTTP-сервера - строка состояния, - /// содержащая три поля: версию HTTP, код состояния и описание. + * first line - response or request + */ + /// Возвращает первую строку ответа HTTP-сервера. + /// @details Первая строка HTTP-сервера - строка состояния, + /// содержащая три поля: версию HTTP, код состояния и описание. const TString& FirstLine() const noexcept; - - /* - * connection can be keep-alive - */ - /// Проверяет, не завершено ли соединение с сервером. - /// @details Транзакция считается завершенной, если не передан заголовок - /// "Connection: Keep Alive". + + /* + * connection can be keep-alive + */ + /// Проверяет, не завершено ли соединение с сервером. + /// @details Транзакция считается завершенной, если не передан заголовок + /// "Connection: Keep Alive". bool IsKeepAlive() const noexcept; - - /* - * output data can be encoded - */ - /// Проверяет, поддерживается ли данный тип кодирования содержимого - /// ответа HTTP-сервера. + + /* + * output data can be encoded + */ + /// Проверяет, поддерживается ли данный тип кодирования содержимого + /// ответа HTTP-сервера. bool AcceptEncoding(const TString& coding) const; - /// Пытается определить наилучший тип кодирования ответа HTTP-сервера. - /// @details Если ответ сервера говорит о том, что поддерживаются - /// любые типы кодирования, выбирается gzip. В противном случае - /// из списка типов кодирования выбирается лучший из поддерживаемых сервером. + /// Пытается определить наилучший тип кодирования ответа HTTP-сервера. + /// @details Если ответ сервера говорит о том, что поддерживаются + /// любые типы кодирования, выбирается gzip. В противном случае + /// из списка типов кодирования выбирается лучший из поддерживаемых сервером. TString BestCompressionScheme() const; TString BestCompressionScheme(TArrayRef<const TStringBuf> codings) const; - - /// Если заголовки содержат Content-Length, возвращает true и - /// записывает значение из заголовка в value + + /// Если заголовки содержат Content-Length, возвращает true и + /// записывает значение из заголовка в value bool GetContentLength(ui64& value) const noexcept; - /// Признак запакованности данных, - если выставлен, то Content-Length, при наличии в заголовках, - /// показывает объём запакованных данных, а из THttpInput мы будем вычитывать уже распакованные. + /// Признак запакованности данных, - если выставлен, то Content-Length, при наличии в заголовках, + /// показывает объём запакованных данных, а из THttpInput мы будем вычитывать уже распакованные. bool ContentEncoded() const noexcept; - + /// Returns true if Content-Length or Transfer-Encoding header received bool HasContent() const noexcept; bool HasExpect100Continue() const noexcept; -private: +private: size_t DoRead(void* buf, size_t len) override; size_t DoSkip(size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /// Передача запроса HTTP-серверу. class THttpOutput: public IOutputStream { -public: +public: THttpOutput(IOutputStream* slave); THttpOutput(IOutputStream* slave, THttpInput* request); ~THttpOutput() override; - - /* - * sent http headers - */ - /// Возвращает контейнер с заголовками запроса к HTTP-серверу. + + /* + * sent http headers + */ + /// Возвращает контейнер с заголовками запроса к HTTP-серверу. const THttpHeaders& SentHeaders() const noexcept; - - /// Устанавливает режим, при котором сервер выдает ответ в упакованном виде. - void EnableCompression(bool enable); + + /// Устанавливает режим, при котором сервер выдает ответ в упакованном виде. + void EnableCompression(bool enable); void EnableCompression(TArrayRef<const TStringBuf> schemas); - /// Устанавливает режим, при котором соединение с сервером не завершается - /// после окончания транзакции. - void EnableKeepAlive(bool enable); - + /// Устанавливает режим, при котором соединение с сервером не завершается + /// после окончания транзакции. + void EnableKeepAlive(bool enable); + /// Устанавливает режим, при котором тело HTTP-запроса/ответа преобразуется в соответствии /// с заголовками Content-Encoding и Transfer-Encoding (включен по умолчанию) void EnableBodyEncoding(bool enable); @@ -124,12 +124,12 @@ public: /// указанным в Content-Encoding (включен по умолчанию) void EnableCompressionHeader(bool enable); - /// Проверяет, производится ли выдача ответов в упакованном виде. + /// Проверяет, производится ли выдача ответов в упакованном виде. bool IsCompressionEnabled() const noexcept; - /// Проверяет, не завершается ли соединение с сервером после окончания транзакции. + /// Проверяет, не завершается ли соединение с сервером после окончания транзакции. bool IsKeepAliveEnabled() const noexcept; - + /// Проверяет, преобразуется ли тело HTTP-запроса/ответа в соответствии /// с заголовками Content-Encoding и Transfer-Encoding bool IsBodyEncodingEnabled() const noexcept; @@ -138,13 +138,13 @@ public: /// указанным в Content-Encoding bool IsCompressionHeaderEnabled() const noexcept; - /* - * is this connection can be really keep-alive - */ - /// Проверяет, можно ли установить режим, при котором соединение с сервером - /// не завершается после окончания транзакции. + /* + * is this connection can be really keep-alive + */ + /// Проверяет, можно ли установить режим, при котором соединение с сервером + /// не завершается после окончания транзакции. bool CanBeKeepAlive() const noexcept; - + void SendContinue(); /* @@ -157,22 +157,22 @@ public: /// учёта chunked transfer encoding) size_t SentSize() const noexcept; -private: +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /// Возвращает код состояния из ответа сервера. -unsigned ParseHttpRetCode(const TStringBuf& ret); - +unsigned ParseHttpRetCode(const TStringBuf& ret); + /// Отправляет HTTP-серверу запрос с минимумом необходимых заголовков. -void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent = "YandexSomething/1.0", const TStringBuf& from = "webadmin@yandex.ru"); +void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent = "YandexSomething/1.0", const TStringBuf& from = "webadmin@yandex.ru"); TArrayRef<const TStringBuf> SupportedCodings(); - + /// @} diff --git a/library/cpp/http/io/stream_ut.cpp b/library/cpp/http/io/stream_ut.cpp index 1ea35df675..b597320405 100644 --- a/library/cpp/http/io/stream_ut.cpp +++ b/library/cpp/http/io/stream_ut.cpp @@ -1,19 +1,19 @@ -#include "stream.h" +#include "stream.h" #include "chunk.h" - + #include <library/cpp/http/server/http_ex.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/string/printf.h> -#include <util/network/socket.h> + +#include <util/string/printf.h> +#include <util/network/socket.h> #include <util/stream/file.h> #include <util/stream/output.h> #include <util/stream/tee.h> -#include <util/stream/zlib.h> +#include <util/stream/zlib.h> #include <util/stream/null.h> - + Y_UNIT_TEST_SUITE(THttpStreamTest) { class TTestHttpServer: public THttpServer::ICallBack { class TRequest: public THttpClientRequestEx { @@ -34,12 +34,12 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { // "lo" is for "local" if (RD.ServerName() == "yandex.lo") { // do redirect - Output() << "HTTP/1.1 301 Moved permanently\r\n" - "Location: http://www.yandex.lo\r\n" - "\r\n"; + Output() << "HTTP/1.1 301 Moved permanently\r\n" + "Location: http://www.yandex.lo\r\n" + "\r\n"; } else if (RD.ServerName() == "www.yandex.lo") { - Output() << "HTTP/1.1 200 Ok\r\n" - "\r\n"; + Output() << "HTTP/1.1 200 Ok\r\n" + "\r\n"; } else { Output() << "HTTP/1.1 200 Ok\r\n\r\n"; if (Buf.Size()) { @@ -80,7 +80,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestCodings1) { UNIT_ASSERT(SupportedCodings().size() > 0); - } + } Y_UNIT_TEST(TestHttpInput) { TString res = "I'm a teapot"; @@ -93,41 +93,41 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { UNIT_ASSERT(server.Start()); TNetworkAddress addr("localhost", port); - TSocket s(addr); - - //TDebugOutput dbg; - TNullOutput dbg; - - { - TSocketOutput so(s); - TTeeOutput out(&so, &dbg); - THttpOutput output(&out); - - output.EnableKeepAlive(true); - output.EnableCompression(true); - + TSocket s(addr); + + //TDebugOutput dbg; + TNullOutput dbg; + + { + TSocketOutput so(s); + TTeeOutput out(&so, &dbg); + THttpOutput output(&out); + + output.EnableKeepAlive(true); + output.EnableCompression(true); + TString r; - r += "GET / HTTP/1.1"; - r += "\r\n"; + r += "GET / HTTP/1.1"; + r += "\r\n"; r += "Host: yandex.lo"; - r += "\r\n"; - r += "\r\n"; - + r += "\r\n"; + r += "\r\n"; + output.Write(r.data(), r.size()); - output.Finish(); - } - - { - TSocketInput si(s); - THttpInput input(&si); - unsigned httpCode = ParseHttpRetCode(input.FirstLine()); - UNIT_ASSERT_VALUES_EQUAL(httpCode / 10, 30u); - - TransferData(&input, &dbg); - } + output.Finish(); + } + + { + TSocketInput si(s); + THttpInput input(&si); + unsigned httpCode = ParseHttpRetCode(input.FirstLine()); + UNIT_ASSERT_VALUES_EQUAL(httpCode / 10, 30u); + + TransferData(&input, &dbg); + } server.Stop(); } - + Y_UNIT_TEST(TestHttpInputDelete) { TString res = "I'm a teapot"; TPortManager pm; @@ -175,9 +175,9 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { } Y_UNIT_TEST(TestParseHttpRetCode) { - UNIT_ASSERT_VALUES_EQUAL(ParseHttpRetCode("HTTP/1.1 301"), 301u); - } - + UNIT_ASSERT_VALUES_EQUAL(ParseHttpRetCode("HTTP/1.1 301"), 301u); + } + Y_UNIT_TEST(TestKeepAlive) { { TString s = "GET / HTTP/1.0\r\n\r\n"; @@ -248,19 +248,19 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TNetworkAddress addr("localhost", port); - TSocket s(addr); - TNullOutput dbg; - + TSocket s(addr); + TNullOutput dbg; + SendMinimalHttpRequest(s, "www.yandex.lo", "/"); - + TSocketInput si(s); - THttpInput input(&si); + THttpInput input(&si); unsigned httpCode = ParseHttpRetCode(input.FirstLine()); - UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); - - TransferData(&input, &dbg); + UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); + + TransferData(&input, &dbg); server.Stop(); - } + } Y_UNIT_TEST(TestResponseWithBlanks) { TString res = "qqqqqq\r\n\r\nsdasdsad\r\n"; @@ -276,7 +276,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TSocket s(addr); - SendMinimalHttpRequest(s, "www.yandex.ru", "/"); + SendMinimalHttpRequest(s, "www.yandex.ru", "/"); TSocketInput si(s); THttpInput input(&si); @@ -289,70 +289,70 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestOutputFlush) { TString str; - TStringOutput strOut(str); - TBufferedOutput bufOut(&strOut, 8192); - THttpOutput httpOut(&bufOut); - - httpOut.EnableKeepAlive(true); - httpOut.EnableCompression(true); + TStringOutput strOut(str); + TBufferedOutput bufOut(&strOut, 8192); + THttpOutput httpOut(&bufOut); - const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; - httpOut << header; + httpOut.EnableKeepAlive(true); + httpOut.EnableCompression(true); + + const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; + httpOut << header; unsigned curLen = str.size(); - const char* body = "<html>Hello</html>"; - httpOut << body; + const char* body = "<html>Hello</html>"; + httpOut << body; UNIT_ASSERT_VALUES_EQUAL(curLen, str.size()); - httpOut.Flush(); + httpOut.Flush(); UNIT_ASSERT_VALUES_EQUAL(curLen + strlen(body), str.size()); - } + } Y_UNIT_TEST(TestOutputPostFlush) { TString str; TString checkStr; - TStringOutput strOut(str); - TStringOutput checkOut(checkStr); - TBufferedOutput bufOut(&strOut, 8192); - TTeeOutput teeOut(&bufOut, &checkOut); - THttpOutput httpOut(&teeOut); + TStringOutput strOut(str); + TStringOutput checkOut(checkStr); + TBufferedOutput bufOut(&strOut, 8192); + TTeeOutput teeOut(&bufOut, &checkOut); + THttpOutput httpOut(&teeOut); - httpOut.EnableKeepAlive(true); - httpOut.EnableCompression(true); + httpOut.EnableKeepAlive(true); + httpOut.EnableCompression(true); - const char* header = "POST / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; - httpOut << header; + const char* header = "POST / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; + httpOut << header; UNIT_ASSERT_VALUES_EQUAL(str.size(), 0u); - const char* body = "<html>Hello</html>"; - httpOut << body; + const char* body = "<html>Hello</html>"; + httpOut << body; UNIT_ASSERT_VALUES_EQUAL(str.size(), 0u); - httpOut.Flush(); + httpOut.Flush(); UNIT_ASSERT_VALUES_EQUAL(checkStr.size(), str.size()); - } + } TString MakeHttpOutputBody(const char* body, bool encodingEnabled) { TString str; - TStringOutput strOut(str); - { - TBufferedOutput bufOut(&strOut, 8192); - THttpOutput httpOut(&bufOut); + TStringOutput strOut(str); + { + TBufferedOutput bufOut(&strOut, 8192); + THttpOutput httpOut(&bufOut); - httpOut.EnableKeepAlive(true); - httpOut.EnableCompression(true); + httpOut.EnableKeepAlive(true); + httpOut.EnableCompression(true); httpOut.EnableBodyEncoding(encodingEnabled); - httpOut << "POST / HTTP/1.1\r\n"; - httpOut << "Host: yandex.ru\r\n"; - httpOut << "Content-Encoding: gzip\r\n"; - httpOut << "\r\n"; + httpOut << "POST / HTTP/1.1\r\n"; + httpOut << "Host: yandex.ru\r\n"; + httpOut << "Content-Encoding: gzip\r\n"; + httpOut << "\r\n"; UNIT_ASSERT_VALUES_EQUAL(str.size(), 0u); - httpOut << body; - } - const char* bodyDelimiter = "\r\n\r\n"; - size_t bodyPos = str.find(bodyDelimiter); + httpOut << body; + } + const char* bodyDelimiter = "\r\n\r\n"; + size_t bodyPos = str.find(bodyDelimiter); UNIT_ASSERT(bodyPos != TString::npos); return str.substr(bodyPos + strlen(bodyDelimiter)); }; @@ -370,54 +370,54 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { const char* body = "<html>Hello</html>"; UNIT_ASSERT(MakeHttpOutputBody(body, false) == body); UNIT_ASSERT(MakeHttpOutputBody(body, true) == SimulateBodyEncoding(body)); - } - + } + Y_UNIT_TEST(TestOutputFinish) { TString str; - TStringOutput strOut(str); - TBufferedOutput bufOut(&strOut, 8192); + TStringOutput strOut(str); + TBufferedOutput bufOut(&strOut, 8192); THttpOutput httpOut(&bufOut); httpOut.EnableKeepAlive(true); httpOut.EnableCompression(true); - const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; - httpOut << header; + const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; + httpOut << header; unsigned curLen = str.size(); - const char* body = "<html>Hello</html>"; + const char* body = "<html>Hello</html>"; httpOut << body; UNIT_ASSERT_VALUES_EQUAL(curLen, str.size()); - httpOut.Finish(); + httpOut.Finish(); UNIT_ASSERT_VALUES_EQUAL(curLen + strlen(body), str.size()); } Y_UNIT_TEST(TestMultilineHeaders) { - const char* headerLine0 = "HTTP/1.1 200 OK"; - const char* headerLine1 = "Content-Language: en"; - const char* headerLine2 = "Vary: Accept-Encoding, "; - const char* headerLine3 = "\tAccept-Language"; - const char* headerLine4 = "Content-Length: 18"; + const char* headerLine0 = "HTTP/1.1 200 OK"; + const char* headerLine1 = "Content-Language: en"; + const char* headerLine2 = "Vary: Accept-Encoding, "; + const char* headerLine3 = "\tAccept-Language"; + const char* headerLine4 = "Content-Length: 18"; TString endLine("\r\n"); TString r; - r += headerLine0 + endLine; - r += headerLine1 + endLine; - r += headerLine2 + endLine; - r += headerLine3 + endLine; - r += headerLine4 + endLine + endLine; - r += "<html>Hello</html>"; - TStringInput stringInput(r); - THttpInput input(&stringInput); - - const THttpHeaders& httpHeaders = input.Headers(); - UNIT_ASSERT_VALUES_EQUAL(httpHeaders.Count(), 3u); - - THttpHeaders::TConstIterator it = httpHeaders.Begin(); + r += headerLine0 + endLine; + r += headerLine1 + endLine; + r += headerLine2 + endLine; + r += headerLine3 + endLine; + r += headerLine4 + endLine + endLine; + r += "<html>Hello</html>"; + TStringInput stringInput(r); + THttpInput input(&stringInput); + + const THttpHeaders& httpHeaders = input.Headers(); + UNIT_ASSERT_VALUES_EQUAL(httpHeaders.Count(), 3u); + + THttpHeaders::TConstIterator it = httpHeaders.Begin(); UNIT_ASSERT_VALUES_EQUAL(it->ToString(), TString(headerLine1)); UNIT_ASSERT_VALUES_EQUAL((++it)->ToString(), TString::Join(headerLine2, headerLine3)); UNIT_ASSERT_VALUES_EQUAL((++it)->ToString(), TString(headerLine4)); - } + } Y_UNIT_TEST(ContentLengthRemoval) { TMemoryInput request("GET / HTTP/1.1\r\nAccept-Encoding: gzip\r\n\r\n"); @@ -430,8 +430,8 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { httpOut.EnableCompression(true); httpOut << "HTTP/1.1 200 OK\r\n"; char answer[] = "Mary had a little lamb."; - httpOut << "Content-Length: " << strlen(answer) << "\r\n" - "\r\n"; + httpOut << "Content-Length: " << strlen(answer) << "\r\n" + "\r\n"; httpOut << answer; httpOut.Finish(); @@ -541,9 +541,9 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TStringStream request; { THttpOutput httpOutput(&request); - httpOutput << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "\r\n"; + httpOutput << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "\r\n"; httpOutput << "GGLOL"; } { @@ -568,9 +568,9 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestInputHasContent) { { TStringStream request; - request << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "\r\n"; + request << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "\r\n"; request << "HTTPDATA"; TStringInput input(request.Str()); @@ -582,10 +582,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { { TStringStream request; - request << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "Content-Length: 8" - "\r\n\r\n"; + request << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "Content-Length: 8" + "\r\n\r\n"; request << "HTTPDATA"; TStringInput input(request.Str()); @@ -597,10 +597,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { { TStringStream request; - request << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "Transfer-Encoding: chunked" - "\r\n\r\n"; + request << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "Transfer-Encoding: chunked" + "\r\n\r\n"; request << "8\r\nHTTPDATA\r\n0\r\n"; TStringInput input(request.Str()); @@ -612,10 +612,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { } Y_UNIT_TEST(TestHttpInputHeadRequest) { - class THeadOnlyInput: public IInputStream { + class THeadOnlyInput: public IInputStream { public: THeadOnlyInput() = default; - + private: size_t DoRead(void* buf, size_t len) override { if (Eof_) { @@ -635,8 +635,8 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { private: TString Data_{TStringBuf("HEAD / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n")}; - size_t Pos_{0}; - bool Eof_{false}; + size_t Pos_{0}; + bool Eof_{false}; }; THeadOnlyInput input; THttpInput httpInput(&input); @@ -647,10 +647,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestHttpOutputResponseToHeadRequestNoZeroChunk) { TStringStream request; - request << "HEAD / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "Connection: Keep-Alive\r\n" - "\r\n"; + request << "HEAD / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "Connection: Keep-Alive\r\n" + "\r\n"; TStringInput input(request.Str()); THttpInput httpInput(&input); diff --git a/library/cpp/http/io/stream_ut_medium.cpp b/library/cpp/http/io/stream_ut_medium.cpp index 2c125eb21e..e70d47555d 100644 --- a/library/cpp/http/io/stream_ut_medium.cpp +++ b/library/cpp/http/io/stream_ut_medium.cpp @@ -1,54 +1,54 @@ -#include "stream.h" +#include "stream.h" #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/zlib.h> - +#include <util/stream/zlib.h> + Y_UNIT_TEST_SUITE(THttpTestMedium) { Y_UNIT_TEST(TestCodings2) { - TStringBuf data = "aaaaaaaaaaaaaaaaaaaaaaa"; - - for (auto codec : SupportedCodings()) { + TStringBuf data = "aaaaaaaaaaaaaaaaaaaaaaa"; + + for (auto codec : SupportedCodings()) { if (codec == TStringBuf("z-zlib-0")) { - continue; - } - + continue; + } + if (codec == TStringBuf("z-null")) { - continue; - } - - TString s; - - { - TStringOutput so(s); - THttpOutput ho(&so); - TBufferedOutput bo(&ho, 10000); - - bo << "HTTP/1.1 200 Ok\r\n" - << "Connection: close\r\n" - << "Content-Encoding: " << codec << "\r\n\r\n"; - - for (size_t i = 0; i < 100; ++i) { - bo << data; - } - } - - try { + continue; + } + + TString s; + + { + TStringOutput so(s); + THttpOutput ho(&so); + TBufferedOutput bo(&ho, 10000); + + bo << "HTTP/1.1 200 Ok\r\n" + << "Connection: close\r\n" + << "Content-Encoding: " << codec << "\r\n\r\n"; + + for (size_t i = 0; i < 100; ++i) { + bo << data; + } + } + + try { UNIT_ASSERT(s.size() > 10); - UNIT_ASSERT(s.find(data) == TString::npos); - } catch (...) { - Cerr << codec << " " << s << Endl; - - throw; - } - - { - TStringInput si(s); - THttpInput hi(&si); - - auto res = hi.ReadAll(); - - UNIT_ASSERT(res.find(data) == 0); - } - } - } - + UNIT_ASSERT(s.find(data) == TString::npos); + } catch (...) { + Cerr << codec << " " << s << Endl; + + throw; + } + + { + TStringInput si(s); + THttpInput hi(&si); + + auto res = hi.ReadAll(); + + UNIT_ASSERT(res.find(data) == 0); + } + } + } + } // THttpTestMedium suite diff --git a/library/cpp/http/io/ut/medium/ya.make b/library/cpp/http/io/ut/medium/ya.make index 235a23dcd7..24d16b00ea 100644 --- a/library/cpp/http/io/ut/medium/ya.make +++ b/library/cpp/http/io/ut/medium/ya.make @@ -1,11 +1,11 @@ UNITTEST_FOR(library/cpp/http/io) - + SIZE(MEDIUM) OWNER(g:util) - -SRCS( + +SRCS( stream_ut_medium.cpp -) - -END() +) + +END() diff --git a/library/cpp/http/io/ut/ya.make b/library/cpp/http/io/ut/ya.make index 84f6949db3..ea0597ebb6 100644 --- a/library/cpp/http/io/ut/ya.make +++ b/library/cpp/http/io/ut/ya.make @@ -1,16 +1,16 @@ UNITTEST_FOR(library/cpp/http/io) - + OWNER(g:util) - + PEERDIR( library/cpp/http/server ) -SRCS( - chunk_ut.cpp +SRCS( + chunk_ut.cpp compression_ut.cpp headers_ut.cpp - stream_ut.cpp -) - -END() + stream_ut.cpp +) + +END() diff --git a/library/cpp/http/io/ya.make b/library/cpp/http/io/ya.make index dcfbd79885..83b30c8958 100644 --- a/library/cpp/http/io/ya.make +++ b/library/cpp/http/io/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - +LIBRARY() + OWNER( g:util mvel ) - -PEERDIR( + +PEERDIR( library/cpp/blockcodecs library/cpp/streams/brotli library/cpp/streams/bzip2 library/cpp/streams/lzma -) - -SRCS( - chunk.cpp +) + +SRCS( + chunk.cpp compression.cpp - headers.cpp + headers.cpp stream.cpp -) - -END() +) + +END() |