diff options
| author | leo <leo@yandex-team.ru> | 2022-02-10 16:46:40 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:40 +0300 |
| commit | 99609724f661f7e21d1cb08e8d80e87c3632fdb3 (patch) | |
| tree | 49e222ea1c5804306084bb3ae065bb702625360f /library | |
| parent | 980edcd3304699edf9d4e4d6a656e585028e2a72 (diff) | |
| download | ydb-99609724f661f7e21d1cb08e8d80e87c3632fdb3.tar.gz | |
Restoring authorship annotation for <leo@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library')
104 files changed, 2254 insertions, 2254 deletions
diff --git a/library/cpp/actors/dnscachelib/dnscache.cpp b/library/cpp/actors/dnscachelib/dnscache.cpp index ae64e1eea4..649339ddb2 100644 --- a/library/cpp/actors/dnscachelib/dnscache.cpp +++ b/library/cpp/actors/dnscachelib/dnscache.cpp @@ -4,7 +4,7 @@ #include <contrib/libs/c-ares/ares.h> #include <util/system/guard.h> -#include <util/datetime/systime.h> +#include <util/datetime/systime.h> const TDnsCache::THost TDnsCache::NullHost; diff --git a/library/cpp/archive/yarchive_ut.cpp b/library/cpp/archive/yarchive_ut.cpp index a5ba9db26b..602a1cdbbd 100644 --- a/library/cpp/archive/yarchive_ut.cpp +++ b/library/cpp/archive/yarchive_ut.cpp @@ -3,7 +3,7 @@ #include <library/cpp/testing/unittest/registar.h> #include <util/string/cast.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/system/tempfile.h> #include <util/memory/blob.h> @@ -49,7 +49,7 @@ void TArchiveTest::TestCreate() { void TArchiveTest::TestRead() { CreateArchive(); TTempFile tmpFile(ARCHIVE); - TBlob blob = TBlob::FromFileSingleThreaded(ARCHIVE); + TBlob blob = TBlob::FromFileSingleThreaded(ARCHIVE); TArchiveReader r(blob); UNIT_ASSERT_EQUAL(r.Count(), 1000); diff --git a/library/cpp/blockcodecs/core/stream.h b/library/cpp/blockcodecs/core/stream.h index ad4220c6a7..fd44ef88f2 100644 --- a/library/cpp/blockcodecs/core/stream.h +++ b/library/cpp/blockcodecs/core/stream.h @@ -7,7 +7,7 @@ #include <util/generic/buffer.h> namespace NBlockCodecs { - struct ICodec; + struct ICodec; class TCodedOutput: public IOutputStream { public: diff --git a/library/cpp/charset/codepage.cpp b/library/cpp/charset/codepage.cpp index b7205e903c..0431bef31b 100644 --- a/library/cpp/charset/codepage.cpp +++ b/library/cpp/charset/codepage.cpp @@ -132,7 +132,7 @@ private: TData Data; TMemoryPool Pool; - + private: inline void AddNameWithCheck(const TString& name, ECharset code) { if (Data.find(name.c_str()) == Data.end()) { @@ -189,7 +189,7 @@ public: return CODES_UNKNOWN; return it->second; - } + } }; ECharset CharsetByName(TStringBuf name) { diff --git a/library/cpp/charset/codepage.h b/library/cpp/charset/codepage.h index 2bad95ee4d..30a02a4610 100644 --- a/library/cpp/charset/codepage.h +++ b/library/cpp/charset/codepage.h @@ -159,12 +159,12 @@ ECharset CharsetByName(TStringBuf name); ECharset CharsetByNameOrDie(TStringBuf name); inline ECharset CharsetByCodePage(const CodePage* CP) { - return CP->CPEnum; -} + return CP->CPEnum; +} inline const char* NameByCharset(ECharset e) { return ::NCodepagePrivate::TCodepagesMap::Instance().NameByCharset(e); -} +} inline const char* NameByCharsetSafe(ECharset e) { if (CODES_UNKNOWN < e && e < CODES_MAX) @@ -174,8 +174,8 @@ inline const char* NameByCharsetSafe(ECharset e) { } inline const char* NameByCodePage(const CodePage* CP) { - return CP->Names[0]; -} + return CP->Names[0]; +} inline const CodePage* CodePageByName(const char* name) { ECharset code = CharsetByName(name); diff --git a/library/cpp/charset/codepage_ut.cpp b/library/cpp/charset/codepage_ut.cpp index 8d7fb4980a..c3ac3ac478 100644 --- a/library/cpp/charset/codepage_ut.cpp +++ b/library/cpp/charset/codepage_ut.cpp @@ -9,8 +9,8 @@ #if defined(_MSC_VER) #pragma warning(disable : 4309) /*truncation of constant value*/ -#endif - +#endif + namespace { const char yandexUpperCase[] = "\x81\x82\x83\x84\x85\x86\x87" diff --git a/library/cpp/charset/doccodes.h b/library/cpp/charset/doccodes.h index ee5108b658..75c87adf9e 100644 --- a/library/cpp/charset/doccodes.h +++ b/library/cpp/charset/doccodes.h @@ -12,11 +12,11 @@ enum ECharset { CODES_RESERVED_3, // reserved code: use it for new encodings before adding them to the end of the list CODES_WIN_EAST, // [ 7] WINDOWS_1250 WIN PL CODES_ISO_EAST, // [ 8] ISO_8859_2 ISO PL - // our superset of subset of windows-1251 + // our superset of subset of windows-1251 CODES_YANDEX, // [ 9] YANDEX CODES_UTF_16BE, // [10] UTF_16BE CODES_UTF_16LE, // [11] UTF_16LE - // missing standard codepages + // missing standard codepages CODES_IBM855, // [12] IBM_855 CODES_UTF8, // [13] UTF8 CODES_UNKNOWNPLANE, // [14] Unrecognized characters are mapped into the PUA: U+F000..U+F0FF diff --git a/library/cpp/charset/recyr.hh b/library/cpp/charset/recyr.hh index 594451e164..5ec8734bcf 100644 --- a/library/cpp/charset/recyr.hh +++ b/library/cpp/charset/recyr.hh @@ -1,6 +1,6 @@ #pragma once -#include <cstdlib> +#include <cstdlib> #include <util/charset/recode_result.h> #include <util/generic/ptr.h> @@ -49,7 +49,7 @@ inline RECODE_RESULT RecodeFromUnicode(ECharset to, const TCharType* in, char* o inline RECODE_RESULT RecodeFromUnicode(ECharset theEncoding, const wchar16* chars, size_t length, char* bytes, size_t size, size_t* read = nullptr, size_t* written = nullptr) { size_t w = 0, r = 0; - RECODE_RESULT rc = ::RecodeFromUnicode(theEncoding, chars, bytes, length, size, r, w); + RECODE_RESULT rc = ::RecodeFromUnicode(theEncoding, chars, bytes, length, size, r, w); if (read) *read = r; if (written) @@ -93,8 +93,8 @@ inline RECODE_RESULT Recode(ECharset from, ECharset to, const char* in, char* ou res = RecodeFromUnicode(to, wide.Get(), out, wideWritten, outSize, wideRead, outWritten); return res; -} - +} + inline RECODE_RESULT Recode(ECharset from, ECharset to, const char* in, char* out, size_t inSize, size_t outSize) { size_t inRead = 0; size_t outWritten = 0; @@ -156,7 +156,7 @@ inline TString RecodeToHTMLEntities(ECharset from, const TString& in) { res = NCodepagePrivate::_recodeToHTMLEntities(from, in.c_str(), out.begin(), in.length(), out.length(), inRead, outWritten); } if (res != RECODE_OK) { - ythrow yexception() << "Recode to HTML entities failed"; + ythrow yexception() << "Recode to HTML entities failed"; } out.resize(outWritten - 1); diff --git a/library/cpp/charset/ya.make b/library/cpp/charset/ya.make index fff2d160ed..7565566bf0 100644 --- a/library/cpp/charset/ya.make +++ b/library/cpp/charset/ya.make @@ -9,8 +9,8 @@ SRCS( cp_encrec.cpp doccodes.cpp iconv.cpp - recyr.hh - recyr_int.hh + recyr.hh + recyr_int.hh ci_string.cpp wide.cpp ) diff --git a/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp b/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp index 296449710d..679089a11b 100644 --- a/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp +++ b/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp @@ -3,7 +3,7 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> #include <util/string/printf.h> -#include <util/generic/ymath.h> +#include <util/generic/ymath.h> class TGreedyDictTest: public TTestBase { UNIT_TEST_SUITE(TGreedyDictTest); diff --git a/library/cpp/containers/2d_array/2d_array.h b/library/cpp/containers/2d_array/2d_array.h index 090a8aff57..9e24650637 100644 --- a/library/cpp/containers/2d_array/2d_array.h +++ b/library/cpp/containers/2d_array/2d_array.h @@ -1,110 +1,110 @@ #pragma once #include <util/system/yassert.h> -#include <util/generic/algorithm.h> +#include <util/generic/algorithm.h> -#ifdef _DEBUG -template <class T> -struct TBoundCheck { - T* Data; +#ifdef _DEBUG +template <class T> +struct TBoundCheck { + T* Data; size_t Size; TBoundCheck(T* d, size_t s) { - Data = d; - Size = s; - } + Data = d; + Size = s; + } T& operator[](size_t i) const { Y_ASSERT(i >= 0 && i < Size); - return Data[i]; - } + return Data[i]; + } }; -#endif +#endif template <class T> -class TArray2D { -private: - typedef T* PT; - T* Data; - T** PData; +class TArray2D { +private: + typedef T* PT; + T* Data; + T** PData; size_t XSize; size_t YSize; -private: +private: void Copy(const TArray2D& a) { - XSize = a.XSize; - YSize = a.YSize; - Create(); + XSize = a.XSize; + YSize = a.YSize; + Create(); for (size_t i = 0; i < XSize * YSize; i++) - Data[i] = a.Data[i]; - } - void Destroy() { - delete[] Data; - delete[] PData; - } - void Create() { - Data = new T[XSize * YSize]; - PData = new PT[YSize]; + Data[i] = a.Data[i]; + } + void Destroy() { + delete[] Data; + delete[] PData; + } + void Create() { + Data = new T[XSize * YSize]; + PData = new PT[YSize]; for (size_t i = 0; i < YSize; i++) - PData[i] = Data + i * XSize; - } + PData[i] = Data + i * XSize; + } public: TArray2D(size_t xsize = 1, size_t ysize = 1) { - XSize = xsize; - YSize = ysize; - Create(); - } + XSize = xsize; + YSize = ysize; + Create(); + } TArray2D(const TArray2D& a) { - Copy(a); - } + Copy(a); + } TArray2D& operator=(const TArray2D& a) { - Destroy(); - Copy(a); - return *this; - } - ~TArray2D() { - Destroy(); - } + Destroy(); + Copy(a); + return *this; + } + ~TArray2D() { + Destroy(); + } void SetSizes(size_t xsize, size_t ysize) { - if (XSize == xsize && YSize == ysize) - return; - Destroy(); - XSize = xsize; - YSize = ysize; - Create(); - } - void Clear() { + if (XSize == xsize && YSize == ysize) + return; + Destroy(); + XSize = xsize; + YSize = ysize; + Create(); + } + void Clear() { SetSizes(1, 1); - } + } #ifdef _DEBUG TBoundCheck<T> operator[](size_t i) const { Y_ASSERT(i < YSize); - return TBoundCheck<T>(PData[i], XSize); - } + return TBoundCheck<T>(PData[i], XSize); + } #else T* operator[](size_t i) const { Y_ASSERT(i < YSize); - return PData[i]; - } + return PData[i]; + } #endif size_t GetXSize() const { - return XSize; - } + return XSize; + } size_t GetYSize() const { - return YSize; - } - void FillZero() { - memset(Data, 0, sizeof(T) * XSize * YSize); - } - void FillEvery(const T& a) { + return YSize; + } + void FillZero() { + memset(Data, 0, sizeof(T) * XSize * YSize); + } + void FillEvery(const T& a) { for (size_t i = 0; i < XSize * YSize; i++) - Data[i] = a; - } - void Swap(TArray2D& a) { - std::swap(Data, a.Data); - std::swap(PData, a.PData); - std::swap(XSize, a.XSize); - std::swap(YSize, a.YSize); - } + Data[i] = a; + } + void Swap(TArray2D& a) { + std::swap(Data, a.Data); + std::swap(PData, a.PData); + std::swap(XSize, a.XSize); + std::swap(YSize, a.YSize); + } }; template <class T> @@ -121,5 +121,5 @@ inline bool operator==(const TArray2D<T>& a, const TArray2D<T>& b) { template <class T> inline bool operator!=(const TArray2D<T>& a, const TArray2D<T>& b) { - return !(a == b); -} + return !(a == b); +} diff --git a/library/cpp/containers/atomizer/atomizer.h b/library/cpp/containers/atomizer/atomizer.h index f18397c24e..5e40f47ab9 100644 --- a/library/cpp/containers/atomizer/atomizer.h +++ b/library/cpp/containers/atomizer/atomizer.h @@ -16,7 +16,7 @@ class super_atomizer; template <class HashFcn, class EqualTo> class atomizer: public string_hash<ui32, HashFcn, EqualTo> { -private: +private: TVector<const char*> order; public: @@ -36,7 +36,7 @@ public: atomizer() { order.reserve(HASH_SIZE_DEFAULT); } - atomizer(size_type hash_size, pool_size_type pool_size) + atomizer(size_type hash_size, pool_size_type pool_size) : string_hash<ui32, HashFcn, EqualTo>(hash_size, pool_size) { order.reserve(hash_size); @@ -80,63 +80,63 @@ public: order.clear(); } void SaveC2N(FILE* f) const { // we write sorted file - for (ui32 i = 0; i < order.size(); i++) - if (order[i]) + for (ui32 i = 0; i < order.size(); i++) + if (order[i]) fprintf(f, "%d\t%s\n", i + 1, order[i]); - } + } void LoadC2N(FILE* f) { // but can read unsorted one - long k, km = 0; - char buf[1000]; + long k, km = 0; + char buf[1000]; char* s; - while (fgets(buf, 1000, f)) { - k = strtol(buf, &s, 10); + while (fgets(buf, 1000, f)) { + k = strtol(buf, &s, 10); char* endl = strchr(s, '\n'); - if (endl) - *endl = 0; - if (k > 0 && k != LONG_MAX) { + if (endl) + *endl = 0; + if (k > 0 && k != LONG_MAX) { km = Max(km, k); - insert_copy(++s, ui32(k)); - } - } - order.resize(km); + insert_copy(++s, ui32(k)); + } + } + order.resize(km); memset(&order[0], 0, order.size()); // if some atoms are absent for (const_iterator I = this->begin(); I != end(); ++I) - order[(*I).second - 1] = (*I).first; - } -}; + order[(*I).second - 1] = (*I).first; + } +}; -template <class T, class HashFcn, class EqualTo> +template <class T, class HashFcn, class EqualTo> class super_atomizer: public string_hash<ui32, HashFcn, EqualTo> { -private: +private: using TOrder = TVector<std::pair<const char*, T>>; - TOrder order; + TOrder order; -public: +public: using iterator = typename string_hash<ui32, HashFcn, EqualTo>::iterator; using const_iterator = typename string_hash<ui32, HashFcn, EqualTo>::const_iterator; using value_type = typename string_hash<ui32, HashFcn, EqualTo>::value_type; using size_type = typename string_hash<ui32, HashFcn, EqualTo>::size_type; using pool_size_type = typename string_hash<ui32, HashFcn, EqualTo>::pool_size_type; - + using o_iterator = typename TOrder::iterator; using o_const_iterator = typename TOrder::const_iterator; using o_value_type = typename TOrder::value_type; - - using string_hash<ui32, HashFcn, EqualTo>::pool; - using string_hash<ui32, HashFcn, EqualTo>::size; - using string_hash<ui32, HashFcn, EqualTo>::find; - using string_hash<ui32, HashFcn, EqualTo>::end; - using string_hash<ui32, HashFcn, EqualTo>::insert_copy; - using string_hash<ui32, HashFcn, EqualTo>::clear_hash; - + + using string_hash<ui32, HashFcn, EqualTo>::pool; + using string_hash<ui32, HashFcn, EqualTo>::size; + using string_hash<ui32, HashFcn, EqualTo>::find; + using string_hash<ui32, HashFcn, EqualTo>::end; + using string_hash<ui32, HashFcn, EqualTo>::insert_copy; + using string_hash<ui32, HashFcn, EqualTo>::clear_hash; + super_atomizer() { - order.reserve(HASH_SIZE_DEFAULT); - } - super_atomizer(size_type hash_size, pool_size_type pool_size) + order.reserve(HASH_SIZE_DEFAULT); + } + super_atomizer(size_type hash_size, pool_size_type pool_size) : string_hash<ui32, HashFcn, EqualTo>(hash_size, pool_size) - { - order.reserve(hash_size); - } + { + order.reserve(hash_size); + } ~super_atomizer() = default; ui32 string_to_atom(const char* key, const T* atom_data = NULL) { const char* old_begin = pool.Begin(); @@ -145,56 +145,56 @@ public: if (ins.second) { // new? if (pool.Begin() != old_begin) // repoint? for (typename TOrder::iterator ptr = order.begin(); ptr != order.end(); ++ptr) - if (old_begin <= (*ptr).first && (*ptr).first < old_end) // from old pool? + if (old_begin <= (*ptr).first && (*ptr).first < old_end) // from old pool? (*ptr).first += pool.Begin() - old_begin; order.push_back(std::pair<const char*, T>((*ins.first).first, atom_data ? *atom_data : T())); - } - return (*ins.first).second; - } - + } + return (*ins.first).second; + } + ui32 perm_string_to_atom(const char* key, const T* atom_data = NULL) { value_type val(key, ui32(size() + 1)); std::pair<iterator, bool> ins = this->insert(val); - if (ins.second) + if (ins.second) order.push_back(std::pair<const char*, T>((*ins.first).first, atom_data ? *atom_data : T())); - return (*ins.first).second; // == size()+1 - } + return (*ins.first).second; // == size()+1 + } ui32 find_atom(const char* key) const { - const_iterator it = find(key); - if (it == end()) - return 0; // INVALID_ATOM - else - return (*it).second; - } + const_iterator it = find(key); + if (it == end()) + return 0; // INVALID_ATOM + else + return (*it).second; + } const char* get_atom_name(ui32 atom) const { - if (atom && atom <= size()) + if (atom && atom <= size()) return order[atom - 1].first; return nullptr; - } - const T* get_atom_data(ui32 atom) const { - if (atom && atom <= size()) + } + const T* get_atom_data(ui32 atom) const { + if (atom && atom <= size()) return &order[atom - 1].second; - return NULL; - } - T* get_atom_data(ui32 atom) { - if (atom && atom <= size()) + return NULL; + } + T* get_atom_data(ui32 atom) { + if (atom && atom <= size()) return &order[atom - 1].second; - return NULL; - } - o_iterator o_begin() { - return order.begin(); - } - o_iterator o_end() { - return order.end(); - } - o_const_iterator o_begin() const { - return order.begin(); - } - o_const_iterator o_end() const { - return order.end(); - } + return NULL; + } + o_iterator o_begin() { + return order.begin(); + } + o_iterator o_end() { + return order.end(); + } + o_const_iterator o_begin() const { + return order.begin(); + } + o_const_iterator o_end() const { + return order.end(); + } void clear_atomizer() { - clear_hash(); - order.clear(); - } + clear_hash(); + order.clear(); + } }; diff --git a/library/cpp/containers/comptrie/comptrie_builder.inl b/library/cpp/containers/comptrie/comptrie_builder.inl index a7da790348..f273fa6571 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.inl +++ b/library/cpp/containers/comptrie/comptrie_builder.inl @@ -508,7 +508,7 @@ void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ConvertSymbolArrayTo for (size_t i = 0; i < keylen; ++i) { TSymbol label = key[i]; - for (int j = (int)NCompactTrie::ExtraBits<TSymbol>(); j >= 0; j -= 8) { + for (int j = (int)NCompactTrie::ExtraBits<TSymbol>(); j >= 0; j -= 8) { Y_ASSERT(ckeyptr < buf.Data() + buflen); *(ckeyptr++) = (char)(label >> j); } diff --git a/library/cpp/containers/str_map/str_map.h b/library/cpp/containers/str_map/str_map.h index 997da8d4a0..31b00d1b99 100644 --- a/library/cpp/containers/str_map/str_map.h +++ b/library/cpp/containers/str_map/str_map.h @@ -31,9 +31,9 @@ pool_insert(Map* m, const char* key, const typename Map::mapped_type& data, TBuf return ins; } -#define HASH_SIZE_DEFAULT 100 +#define HASH_SIZE_DEFAULT 100 #define AVERAGEWORD_BUF 10 - + template <class T, class HashFcn, class EqualTo, class Alloc> class string_hash: public THashMap<const char*, T, HashFcn, EqualTo, Alloc> { protected: @@ -49,7 +49,7 @@ public: string_hash() { pool.Reserve(HASH_SIZE_DEFAULT * AVERAGEWORD_BUF); // reserve here } - string_hash(size_type hash_size, pool_size_type pool_size) + string_hash(size_type hash_size, pool_size_type pool_size) : THashMap<const char*, T, HashFcn, EqualTo, Alloc>(hash_size) { pool.Reserve(pool_size); // reserve here @@ -66,46 +66,46 @@ public: pool_size_type pool_size() const { return pool.Size(); } - - string_hash(const string_hash& sh) + + string_hash(const string_hash& sh) : THashMap<const char*, T, HashFcn, EqualTo, Alloc>() - { - for (const_iterator i = sh.begin(); i != sh.end(); ++i) - insert_copy((*i).first, (*i).second); - } - /* May be faster? - string_hash(const string_hash& sh) + { + for (const_iterator i = sh.begin(); i != sh.end(); ++i) + insert_copy((*i).first, (*i).second); + } + /* May be faster? + string_hash(const string_hash& sh) : THashMap<const char *, T, HashFcn, EqualTo>(sh) - { - pool = sh.pool; - size_t delta = pool.begin() - sh.pool.begin(); - for (iterator i = begin(); i != end(); ++i) - (const char*&)(*i).first += delta; - } - */ + { + pool = sh.pool; + size_t delta = pool.begin() - sh.pool.begin(); + for (iterator i = begin(); i != end(); ++i) + (const char*&)(*i).first += delta; + } + */ string_hash& operator=(const string_hash& sh) { - if (&sh != this) { - clear_hash(); - for (const_iterator i = sh.begin(); i != sh.end(); ++i) - insert_copy((*i).first, (*i).second); - } - return *this; - } - + if (&sh != this) { + clear_hash(); + for (const_iterator i = sh.begin(); i != sh.end(); ++i) + insert_copy((*i).first, (*i).second); + } + return *this; + } + mapped_type& operator[](const char* key) { iterator I = yh::find(key); if (I == yh::end()) - I = insert_copy(key, mapped_type()).first; - return (*I).second; - } + I = insert_copy(key, mapped_type()).first; + return (*I).second; + } }; -template <class C, class T, class HashFcn, class EqualTo> +template <class C, class T, class HashFcn, class EqualTo> class THashWithSegmentedPoolForKeys: protected THashMap<const C*, T, HashFcn, EqualTo>, TNonCopyable { -protected: - segmented_pool<C> pool; +protected: + segmented_pool<C> pool; -public: +public: using yh = THashMap<const C*, T, HashFcn, EqualTo>; using iterator = typename yh::iterator; using const_iterator = typename yh::const_iterator; @@ -114,68 +114,68 @@ public: using key_type = typename yh::key_type; using value_type = typename yh::value_type; - THashWithSegmentedPoolForKeys(size_type hash_size = HASH_SIZE_DEFAULT, size_t segsize = HASH_SIZE_DEFAULT * AVERAGEWORD_BUF, bool afs = false) - : yh(hash_size) - , pool(segsize) - { - if (afs) - pool.alloc_first_seg(); - } - + THashWithSegmentedPoolForKeys(size_type hash_size = HASH_SIZE_DEFAULT, size_t segsize = HASH_SIZE_DEFAULT * AVERAGEWORD_BUF, bool afs = false) + : yh(hash_size) + , pool(segsize) + { + if (afs) + pool.alloc_first_seg(); + } + std::pair<iterator, bool> insert_copy(const C* key, size_t keylen, const mapped_type& data) { std::pair<iterator, bool> ins = this->insert(value_type(key, data)); - if (ins.second) // new? - (const C*&)(*ins.first).first = pool.append(key, keylen); - return ins; - } - - void clear_hash() { - yh::clear(); - pool.restart(); - } - - size_t pool_size() const { - return pool.size(); - } - - size_t size() const { - return yh::size(); - } - - bool empty() const { - return yh::empty(); - } - - iterator begin() { - return yh::begin(); - } - - iterator end() { - return yh::end(); - } - - const_iterator begin() const { - return yh::begin(); - } - - const_iterator end() const { - return yh::end(); - } - - iterator find(const key_type& key) { - return yh::find(key); - } - - const_iterator find(const key_type& key) const { - return yh::find(key); - } - + if (ins.second) // new? + (const C*&)(*ins.first).first = pool.append(key, keylen); + return ins; + } + + void clear_hash() { + yh::clear(); + pool.restart(); + } + + size_t pool_size() const { + return pool.size(); + } + + size_t size() const { + return yh::size(); + } + + bool empty() const { + return yh::empty(); + } + + iterator begin() { + return yh::begin(); + } + + iterator end() { + return yh::end(); + } + + const_iterator begin() const { + return yh::begin(); + } + + const_iterator end() const { + return yh::end(); + } + + iterator find(const key_type& key) { + return yh::find(key); + } + + const_iterator find(const key_type& key) const { + return yh::find(key); + } + const yh& get_THashMap() const { return static_cast<const yh&>(*this); } -}; - -template <class T, class HashFcn, class EqualTo> +}; + +template <class T, class HashFcn, class EqualTo> class segmented_string_hash: public THashWithSegmentedPoolForKeys<char, T, HashFcn, EqualTo> { public: using Base = THashWithSegmentedPoolForKeys<char, T, HashFcn, EqualTo>; @@ -186,20 +186,20 @@ public: using key_type = typename Base::key_type; using value_type = typename Base::value_type; -public: - segmented_string_hash(size_type hash_size = HASH_SIZE_DEFAULT, size_t segsize = HASH_SIZE_DEFAULT * AVERAGEWORD_BUF, bool afs = false) +public: + segmented_string_hash(size_type hash_size = HASH_SIZE_DEFAULT, size_t segsize = HASH_SIZE_DEFAULT * AVERAGEWORD_BUF, bool afs = false) : Base(hash_size, segsize, afs) { } - + std::pair<iterator, bool> insert_copy(const char* key, const mapped_type& data) { - return Base::insert_copy(key, strlen(key) + 1, data); - } - - mapped_type& operator[](const char* key) { - iterator I = Base::find(key); - if (I == Base::end()) - I = insert_copy(key, mapped_type()).first; - return (*I).second; + return Base::insert_copy(key, strlen(key) + 1, data); + } + + mapped_type& operator[](const char* key) { + iterator I = Base::find(key); + if (I == Base::end()) + I = insert_copy(key, mapped_type()).first; + return (*I).second; } }; diff --git a/library/cpp/coroutine/engine/coroutine_ut.cpp b/library/cpp/coroutine/engine/coroutine_ut.cpp index d03b157fdf..8b372496a2 100644 --- a/library/cpp/coroutine/engine/coroutine_ut.cpp +++ b/library/cpp/coroutine/engine/coroutine_ut.cpp @@ -282,7 +282,7 @@ void TCoroTest::TestMutex() { TContExecutor e(32000); e.Execute(CoMutexTest); UNIT_ASSERT_EQUAL(res, "1212"); - res.clear(); + res.clear(); } static TContMutex m1; @@ -322,7 +322,7 @@ void TCoroTest::TestCondVar() { TContExecutor e(32000); e.Execute(CoCondVarTest); UNIT_ASSERT_EQUAL(res, "0^1234561^1234562^123456"); - res.clear(); + res.clear(); } namespace NCoroTestJoin { diff --git a/library/cpp/coroutine/engine/poller.h b/library/cpp/coroutine/engine/poller.h index 345bc22888..8ea012c0fc 100644 --- a/library/cpp/coroutine/engine/poller.h +++ b/library/cpp/coroutine/engine/poller.h @@ -18,7 +18,7 @@ enum class EContPoller { class IPollerFace { public: struct TChange { - SOCKET Fd; + SOCKET Fd; void* Data; ui16 Flags; }; diff --git a/library/cpp/coroutine/listener/listen.h b/library/cpp/coroutine/listener/listen.h index 1089cec411..3a89cd3ecc 100644 --- a/library/cpp/coroutine/listener/listen.h +++ b/library/cpp/coroutine/listener/listen.h @@ -3,7 +3,7 @@ #include <util/generic/ptr.h> #include <util/generic/ylimits.h> -struct TIpAddress; +struct TIpAddress; class TContExecutor; class TSocketHolder; class TNetworkAddress; @@ -110,7 +110,7 @@ public: void Stop() noexcept; void StopListenAddr(const NAddr::IRemoteAddr& addr); - void StopListenAddr(const TIpAddress& addr); + void StopListenAddr(const TIpAddress& addr); void StopListenAddr(const TNetworkAddress& addr); template <class T> diff --git a/library/cpp/deprecated/mapped_file/mapped_file.cpp b/library/cpp/deprecated/mapped_file/mapped_file.cpp index 1239b9edca..b0e4511299 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.cpp +++ b/library/cpp/deprecated/mapped_file/mapped_file.cpp @@ -1,6 +1,6 @@ #include "mapped_file.h" -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/system/defaults.h> #include <util/system/hi_lo.h> #include <util/system/filemap.h> @@ -57,8 +57,8 @@ void TMappedFile::init(const TString& name, TFileMap::EOpenMode om) { Y_UNUSED(map.Release()); newFile.swap(*this); newFile.term(); -} - +} + void TMappedFile::flush() { Map_->Flush(); } diff --git a/library/cpp/deprecated/mapped_file/mapped_file.h b/library/cpp/deprecated/mapped_file/mapped_file.h index 4da8aacf55..45859ed65a 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.h +++ b/library/cpp/deprecated/mapped_file/mapped_file.h @@ -4,7 +4,7 @@ #include <util/generic/ptr.h> #include <util/generic/string.h> #include <util/generic/utility.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/system/align.h> #include <util/system/file.h> #include <util/system/filemap.h> @@ -15,25 +15,25 @@ /// Deprecated (by pg@), use TFileMap or TMemoryMap instead class TMappedFile { -private: +private: TFileMap* Map_; private: TMappedFile(TFileMap* map, const char* dbgName); -public: +public: TMappedFile() { Map_ = nullptr; - } + } ~TMappedFile() { - term(); - } + term(); + } explicit TMappedFile(const TString& name) { Map_ = nullptr; init(name, TFileMap::oRdOnly); - } + } TMappedFile(const TFile& file, TFileMap::EOpenMode om = TFileMap::oRdOnly, const char* dbgName = "unknown"); @@ -52,8 +52,8 @@ public: Map_->Unmap(); delete Map_; Map_ = nullptr; - } - } + } + } size_t getSize() const { return (Map_ ? Map_->MappedSize() : 0); @@ -69,4 +69,4 @@ public: void swap(TMappedFile& file) noexcept { DoSwap(Map_, file.Map_); } -}; +}; diff --git a/library/cpp/digest/md5/md5.cpp b/library/cpp/digest/md5/md5.cpp index 4d6168c4e8..24a5b69eef 100644 --- a/library/cpp/digest/md5/md5.cpp +++ b/library/cpp/digest/md5/md5.cpp @@ -5,7 +5,7 @@ #include <util/stream/input.h> #include <util/stream/file.h> #include <util/string/hex.h> - + #include <contrib/libs/nayuki_md5/md5.h> namespace { @@ -42,11 +42,11 @@ char* MD5::File(const char* filename, char* buf) { return Stream(&fi, buf); } catch (...) { - } + } return nullptr; -} - +} + TString MD5::File(const TString& filename) { TString buf; buf.ReserveAndResize(MD5_HEX_DIGEST_LENGTH); @@ -63,8 +63,8 @@ char* MD5::Data(const TArrayRef<const ui8>& data, char* buf) { char* MD5::Data(const void* data, size_t len, char* buf) { return Data(MakeUnsignedArrayRef(data, len), buf); -} - +} + TString MD5::Data(const TArrayRef<const ui8>& data) { TString buf; buf.ReserveAndResize(MD5_HEX_DIGEST_LENGTH); @@ -92,25 +92,25 @@ static const ui8 PADDING[MD5_BLOCK_LENGTH] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/* MD5 initialization. Begins an MD5 operation, writing a new context. */ - + +/* MD5 initialization. Begins an MD5 operation, writing a new context. */ + void MD5::Init() { BufferSize = 0; StreamSize = 0; - /* Load magic initialization constants. */ + /* Load magic initialization constants. */ State[0] = 0x67452301; State[1] = 0xefcdab89; State[2] = 0x98badcfe; State[3] = 0x10325476; -} - +} + /* - * MD5 block update operation. Continues an MD5 message-digest - * operation, processing another message block, and updating the - * context. - */ - + * MD5 block update operation. Continues an MD5 message-digest + * operation, processing another message block, and updating the + * context. + */ + void MD5::UpdatePart(TArrayRef<const ui8> data) { /* Count input bytes */ StreamSize += data.size(); @@ -135,12 +135,12 @@ void MD5::UpdatePart(TArrayRef<const ui8> data) { /* Save remaining input in buffer */ memcpy(Buffer, data.data(), data.size()); BufferSize += data.size(); -} - -/* - * MD5 padding. Adds padding followed by original length. - */ - +} + +/* + * MD5 padding. Adds padding followed by original length. + */ + void MD5::Pad() { size_t streamSize = StreamSize; @@ -155,40 +155,40 @@ void MD5::Pad() { Buffer[MD5_PADDING_SHIFT + i] = static_cast<ui8>(streamSize & 0xFFU); } md5_compress(State, Buffer); -} - -/* - * MD5 finalization. Ends an MD5 message-digest operation, writing the - * the message digest and zeroizing the context. - */ - +} + +/* + * MD5 finalization. Ends an MD5 message-digest operation, writing the + * the message digest and zeroizing the context. + */ + ui8* MD5::Final(ui8 digest[16]) { - /* Do padding. */ - Pad(); - /* Store state in digest */ + /* Do padding. */ + Pad(); + /* Store state in digest */ memcpy(digest, State, 16); - /* Zeroize sensitive information. */ + /* Zeroize sensitive information. */ Init(); return digest; -} - +} + char* MD5::End(char* buf) { static const char hex[] = "0123456789abcdef"; ui8 digest[16]; if (!buf) - buf = (char*)malloc(33); + buf = (char*)malloc(33); if (!buf) return nullptr; - Final(digest); + Final(digest); for (ui8 i = 0; i < MD5_HEX_DIGEST_LENGTH / 2; i++) { buf[i * 2] = hex[digest[i] >> 4]; buf[i * 2 + 1] = hex[digest[i] & 0x0f]; - } + } buf[32] = '\0'; - return buf; -} - + return buf; +} + char* MD5::End_b64(char* buf) { ui8 digest[16]; if (!buf) diff --git a/library/cpp/digest/md5/md5.h b/library/cpp/digest/md5/md5.h index ead7158a6e..2c17aa0518 100644 --- a/library/cpp/digest/md5/md5.h +++ b/library/cpp/digest/md5/md5.h @@ -1,8 +1,8 @@ #pragma once - + #include <util/generic/array_ref.h> #include <util/generic/strbuf.h> - + class IInputStream; class MD5 { @@ -74,4 +74,4 @@ private: ui8 Buffer[64]; /* input buffer */ ui32 State[4]; /* state (ABCD) */ ui64 StreamSize; /* total bytes in input stream */ -}; +}; diff --git a/library/cpp/digest/old_crc/crc.h b/library/cpp/digest/old_crc/crc.h index a737f2e8b7..4a3ce6d05e 100644 --- a/library/cpp/digest/old_crc/crc.h +++ b/library/cpp/digest/old_crc/crc.h @@ -46,8 +46,8 @@ inline ui32 crc32(const char* buf, size_t buflen, ui32 crcinit = CRC32INIT) { inline ui32 crc32(const void* buf, size_t buflen, ui32 crcinit = CRC32INIT) { return crc32((const char*)buf, buflen, crcinit); -} - +} + // Copyright (C) Sewell Development Corporation, 1994 - 1998. inline ui64 crc64(const void* buf, size_t buflen, ui64 crcinit = CRC64INIT) { const unsigned char* ptr = (const unsigned char*)buf; diff --git a/library/cpp/digest/old_crc/gencrc/main.cpp b/library/cpp/digest/old_crc/gencrc/main.cpp index a7e3c25561..d5821304ce 100644 --- a/library/cpp/digest/old_crc/gencrc/main.cpp +++ b/library/cpp/digest/old_crc/gencrc/main.cpp @@ -2,7 +2,7 @@ #define POLY_16 0x1021 #define POLY_32 0xEDB88320UL -#define POLY_64 ULL(0xE543279765927881) +#define POLY_64 ULL(0xE543279765927881) static void crc16init() { ui32 CRCTAB16[256]; diff --git a/library/cpp/digest/sfh/sfh.h b/library/cpp/digest/sfh/sfh.h index aa48f3ceeb..372938654c 100644 --- a/library/cpp/digest/sfh/sfh.h +++ b/library/cpp/digest/sfh/sfh.h @@ -6,9 +6,9 @@ inline ui32 SuperFastHash(const void* d, size_t l) noexcept { ui32 hash = (ui32)l; ui32 tmp; - + if (!l || !d) - return 0; + return 0; TUnalignedMemoryIterator<ui16, 4> iter(d, l); diff --git a/library/cpp/getopt/small/opt.cpp b/library/cpp/getopt/small/opt.cpp index f54e008f79..744501765c 100644 --- a/library/cpp/getopt/small/opt.cpp +++ b/library/cpp/getopt/small/opt.cpp @@ -63,7 +63,7 @@ int Opt::Get(int* longOptionIndex) { try { bool r = OptsParser_->Next(); - Ind = (int)OptsParser_->Pos_; + Ind = (int)OptsParser_->Pos_; if (!r) { return EOF; } else { @@ -74,7 +74,7 @@ int Opt::Get(int* longOptionIndex) { } else { const Ion* ion = (const Ion*)OptsParser_->CurOpt()->UserValue(); if (longOptionIndex) { - *longOptionIndex = int(ion - Ions_); + *longOptionIndex = int(ion - Ions_); } char c = OptsParser_->CurOpt()->GetCharOr0(); return c != 0 ? c : ion->val; @@ -93,7 +93,7 @@ void Opt::DummyHelp(IOutputStream& os) { } int Opt::GetArgC() const { - return (int)OptsParser_->Argc_; + return (int)OptsParser_->Argc_; } const char** Opt::GetArgV() const { diff --git a/library/cpp/getopt/small/opt2.cpp b/library/cpp/getopt/small/opt2.cpp index 0f6eb6d179..0cdc774e78 100644 --- a/library/cpp/getopt/small/opt2.cpp +++ b/library/cpp/getopt/small/opt2.cpp @@ -45,7 +45,7 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { if (*s == '?' || *s == '-') ythrow yexception() << "Opt2: Symbol '" << *s << "' can not be used in optspec because it is reserved"; Specs.push_back(Opt2Param()); - SpecsMap[(ui8)*s] = (ui8)Specs.size(); // actual index + 1 + SpecsMap[(ui8)*s] = (ui8)Specs.size(); // actual index + 1 Specs.back().opt = *s; if (s[1] == ':') { Specs.back().HasArg = true; diff --git a/library/cpp/getopt/small/posix_getopt.cpp b/library/cpp/getopt/small/posix_getopt.cpp index c8cab678ac..bd06f3499f 100644 --- a/library/cpp/getopt/small/posix_getopt.cpp +++ b/library/cpp/getopt/small/posix_getopt.cpp @@ -48,7 +48,7 @@ namespace NLastGetopt { return -1; } else { optarg = (char*)OptsParser->CurVal(); - optind = (int)OptsParser->Pos_; + optind = (int)OptsParser->Pos_; if (longindex && OptsParser->CurOpt()) *longindex = (int)Opts->IndexOf(OptsParser->CurOpt()); return OptsParser->CurOpt() ? OptsParser->CurOpt()->GetCharOr0() : 1; diff --git a/library/cpp/getopt/ut/opt2_ut.cpp b/library/cpp/getopt/ut/opt2_ut.cpp index 4f6ac64e29..0e7464747c 100644 --- a/library/cpp/getopt/ut/opt2_ut.cpp +++ b/library/cpp/getopt/ut/opt2_ut.cpp @@ -36,9 +36,9 @@ Y_UNIT_TEST_SUITE(Opt2Test) { UNIT_ASSERT_VALUES_EQUAL((void*)edef, e); UNIT_ASSERT_VALUES_EQUAL(2u, opt.Pos.size()); - UNIT_ASSERT_STRINGS_EQUAL("zz", opt.Pos.at(0)); + UNIT_ASSERT_STRINGS_EQUAL("zz", opt.Pos.at(0)); UNIT_ASSERT_VALUES_EQUAL((void*)argv[2], opt.Pos.at(0)); - UNIT_ASSERT_STRINGS_EQUAL("ww", opt.Pos.at(1)); + UNIT_ASSERT_STRINGS_EQUAL("ww", opt.Pos.at(1)); UNIT_ASSERT_STRINGS_EQUAL("1", x.at(0)); UNIT_ASSERT_STRINGS_EQUAL("2", x.at(1)); } diff --git a/library/cpp/http/fetch/exthttpcodes.cpp b/library/cpp/http/fetch/exthttpcodes.cpp index ac0d3b359d..acc05650c8 100644 --- a/library/cpp/http/fetch/exthttpcodes.cpp +++ b/library/cpp/http/fetch/exthttpcodes.cpp @@ -1,18 +1,18 @@ #include "exthttpcodes.h" -#include <cstring> - +#include <cstring> + const ui16 CrazyServer = ShouldDelete | MarkSuspect; - + struct http_flag { ui16 http; ui16 flag; }; -static http_flag HTTP_FLAG[] = { +static http_flag HTTP_FLAG[] = { {HTTP_CONTINUE, MarkSuspect}, // 100 {HTTP_SWITCHING_PROTOCOLS, CrazyServer}, // 101 {HTTP_PROCESSING, CrazyServer}, // 102 - + {HTTP_OK, ShouldReindex}, // 200 {HTTP_CREATED, CrazyServer}, // 201 {HTTP_ACCEPTED, ShouldDelete}, // 202 @@ -23,7 +23,7 @@ static http_flag HTTP_FLAG[] = { {HTTP_MULTI_STATUS, CrazyServer}, // 207 {HTTP_ALREADY_REPORTED, CrazyServer}, // 208 {HTTP_IM_USED, CrazyServer}, // 226 - + {HTTP_MULTIPLE_CHOICES, CheckLinks | ShouldDelete}, // 300 {HTTP_MOVED_PERMANENTLY, CheckLocation | ShouldDelete | MoveRedir}, // 301 {HTTP_FOUND, CheckLocation | ShouldDelete | MoveRedir}, // 302 @@ -32,7 +32,7 @@ static http_flag HTTP_FLAG[] = { {HTTP_USE_PROXY, ShouldDelete}, // 305 {HTTP_TEMPORARY_REDIRECT, CheckLocation | ShouldDelete | MoveRedir}, // 307 {HTTP_PERMANENT_REDIRECT, CheckLocation | ShouldDelete | MoveRedir}, // 308 - + {HTTP_BAD_REQUEST, CrazyServer}, // 400 {HTTP_UNAUTHORIZED, ShouldDelete}, // 401 {HTTP_PAYMENT_REQUIRED, ShouldDelete}, // 402 @@ -53,7 +53,7 @@ static http_flag HTTP_FLAG[] = { {HTTP_EXPECTATION_FAILED, ShouldDelete}, // 417 {HTTP_I_AM_A_TEAPOT, CrazyServer}, // 418 {HTTP_AUTHENTICATION_TIMEOUT, ShouldDelete}, // 419 - + {HTTP_MISDIRECTED_REQUEST, CrazyServer}, // 421 {HTTP_UNPROCESSABLE_ENTITY, CrazyServer}, // 422 {HTTP_LOCKED, ShouldDelete}, // 423 @@ -62,7 +62,7 @@ static http_flag HTTP_FLAG[] = { {HTTP_PRECONDITION_REQUIRED, ShouldDelete}, // 428 {HTTP_TOO_MANY_REQUESTS, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 429 {HTTP_UNAVAILABLE_FOR_LEGAL_REASONS, ShouldDelete}, // 451 - + {HTTP_INTERNAL_SERVER_ERROR, MarkSuspect}, // 500 {HTTP_NOT_IMPLEMENTED, ShouldDelete | ShouldDisconnect}, // 501 {HTTP_BAD_GATEWAY, MarkSuspect}, // 502 @@ -116,7 +116,7 @@ static http_flag HTTP_FLAG[] = { {HTTP_FETCHER_BAD_RESPONSE, 0}, // 1040 {HTTP_FETCHER_MB_ERROR, 0}, // 1041 {HTTP_SSL_CERT_ERROR, 0}, // 1042 - + // Custom (replace HTTP 200/304) {EXT_HTTP_MIRRMOVE, 0}, // 2000 {EXT_HTTP_MANUAL_DELETE, ShouldDelete}, // 2001 @@ -142,34 +142,34 @@ static http_flag HTTP_FLAG[] = { {EXT_HTTP_EMPTY_RESPONSE, ShouldDelete}, // 2024 {EXT_HTTP_REL_CANONICAL, ShouldDelete | CheckLinks | MoveRedir}, // 2025 {0, 0}}; - + static ui16* prepare_flags(http_flag* arg) { - static ui16 flags[EXT_HTTP_CODE_MAX]; + static ui16 flags[EXT_HTTP_CODE_MAX]; http_flag* ptr; - size_t i; - + size_t i; + // устанавливаем значение по умолчанию для кодов не перечисленных в таблице выше for (i = 0; i < EXT_HTTP_CODE_MAX; ++i) - flags[i] = CrazyServer; - + flags[i] = CrazyServer; + // устанавливаем флаги для перечисленных кодов for (ptr = arg; ptr->http; ++ptr) flags[ptr->http & (EXT_HTTP_CODE_MAX - 1)] = ptr->flag; - + // для стандартных кодов ошибок берем флаги из первого кода каждой группы и проставляем их // всем кодам не перечисленным в таблице выше for (size_t group = 0; group < 1000; group += 100) for (size_t j = group + 1; j < group + 100; ++j) flags[j] = flags[group]; - + // предыдущий цикл затер некоторые флаги перечисленные в таблице выше // восстанавливаем их for (ptr = arg; ptr->http; ++ptr) flags[ptr->http & (EXT_HTTP_CODE_MAX - 1)] = ptr->flag; - - return flags; -} - + + return flags; +} + ui16* http2status = prepare_flags(HTTP_FLAG); TStringBuf ExtHttpCodeStr(int code) noexcept { diff --git a/library/cpp/http/fetch/exthttpcodes.h b/library/cpp/http/fetch/exthttpcodes.h index 88bfe8d829..6b525052cd 100644 --- a/library/cpp/http/fetch/exthttpcodes.h +++ b/library/cpp/http/fetch/exthttpcodes.h @@ -1,9 +1,9 @@ #pragma once - -#include <util/system/defaults.h> + +#include <util/system/defaults.h> #include <library/cpp/http/misc/httpcodes.h> - -enum ExtHttpCodes { + +enum ExtHttpCodes { // Custom HTTP_EXTENDED = 1000, HTTP_BAD_RESPONSE_HEADER = 1000, @@ -50,8 +50,8 @@ enum ExtHttpCodes { HTTP_FETCHER_MB_ERROR = 1041, HTTP_SSL_CERT_ERROR = 1042, HTTP_PROXY_REQUEST_CANCELED = 1051, - - // Custom (replace HTTP 200/304) + + // Custom (replace HTTP 200/304) EXT_HTTP_EXT_SUCCESS_BEGIN = 2000, // to check if code variable is in success interval EXT_HTTP_MIRRMOVE = 2000, EXT_HTTP_MANUAL_DELETE = 2001, @@ -104,22 +104,22 @@ enum ExtHttpCodes { EXT_HTTP_WRONGMULTILANG = 3023, EXT_HTTP_SOFTMIRRORS = 3024, EXT_HTTP_BIGLEVEL = 3025, - - // fast robot codes - + + // fast robot codes + EXT_HTTP_FASTHOPS = 4000, EXT_HTTP_NODOC = 4001, EXT_HTTP_MAX -}; - -enum HttpFlags { - // connection - ShouldDisconnect = 1, +}; + +enum HttpFlags { + // connection + ShouldDisconnect = 1, ShouldRetry = 2, // UNUSED 4 - - // indexer + + // indexer ShouldReindex = 8, ShouldDelete = 16, CheckLocation = 32, @@ -129,13 +129,13 @@ enum HttpFlags { // UNUSED 512 MoveRedir = 1024, CanBeFake = 2048, -}; - +}; + const size_t EXT_HTTP_CODE_MAX = 1 << 12; - -static inline int Http2Status(int code) { + +static inline int Http2Status(int code) { extern ui16* http2status; return http2status[code & (EXT_HTTP_CODE_MAX - 1)]; -} +} TStringBuf ExtHttpCodeStr(int code) noexcept; diff --git a/library/cpp/http/fetch/http_digest.cpp b/library/cpp/http/fetch/http_digest.cpp index 1c8bc6f449..1eaa02b7f2 100644 --- a/library/cpp/http/fetch/http_digest.cpp +++ b/library/cpp/http/fetch/http_digest.cpp @@ -3,204 +3,204 @@ #include <library/cpp/digest/md5/md5.h> #include <util/stream/output.h> #include <util/stream/str.h> - -/************************************************************/ -/************************************************************/ -static const char* WWW_PREFIX = "Authorization: Digest "; - -/************************************************************/ + +/************************************************************/ +/************************************************************/ +static const char* WWW_PREFIX = "Authorization: Digest "; + +/************************************************************/ httpDigestHandler::httpDigestHandler() : User_(nullptr) , Password_(nullptr) , Nonce_(nullptr) , NonceCount_(0) , HeaderInstruction_(nullptr) -{ -} - -/************************************************************/ +{ +} + +/************************************************************/ httpDigestHandler::~httpDigestHandler() { - clear(); -} - -/************************************************************/ + clear(); +} + +/************************************************************/ void httpDigestHandler::clear() { free(Nonce_); free(HeaderInstruction_); User_ = Password_ = nullptr; Nonce_ = HeaderInstruction_ = nullptr; NonceCount_ = 0; -} - -/************************************************************/ +} + +/************************************************************/ void httpDigestHandler::setAuthorization(const char* user, const char* password) { - clear(); + clear(); if (user && password) { User_ = user; Password_ = password; - } -} - -/************************************************************/ + } +} + +/************************************************************/ const char* httpDigestHandler::getHeaderInstruction() const { return HeaderInstruction_; -} - -/************************************************************/ +} + +/************************************************************/ void httpDigestHandler::generateCNonce(char* outCNonce) { - if (!*outCNonce) + if (!*outCNonce) sprintf(outCNonce, "%ld", (long)time(nullptr)); -} - -/************************************************************/ +} + +/************************************************************/ inline void addMD5(MD5& ctx, const char* value) { - ctx.Update((const unsigned char*)(value), strlen(value)); -} - + ctx.Update((const unsigned char*)(value), strlen(value)); +} + inline void addMD5(MD5& ctx, const char* value, int len) { - ctx.Update((const unsigned char*)(value), len); -} - + ctx.Update((const unsigned char*)(value), len); +} + inline void addMD5Sep(MD5& ctx) { - addMD5(ctx, ":", 1); -} - -/************************************************************/ -/* calculate H(A1) as per spec */ + addMD5(ctx, ":", 1); +} + +/************************************************************/ +/* calculate H(A1) as per spec */ void httpDigestHandler::digestCalcHA1(const THttpAuthHeader& hd, char* outSessionKey, char* outCNonce) { - MD5 ctx; - ctx.Init(); + MD5 ctx; + ctx.Init(); addMD5(ctx, User_); addMD5Sep(ctx); addMD5(ctx, hd.realm); addMD5Sep(ctx); addMD5(ctx, Password_); - + if (hd.algorithm == 1) { //MD5-sess - unsigned char digest[16]; - ctx.Final(digest); - - generateCNonce(outCNonce); - - ctx.Init(); - ctx.Update(digest, 16); + unsigned char digest[16]; + ctx.Final(digest); + + generateCNonce(outCNonce); + + ctx.Init(); + ctx.Update(digest, 16); addMD5Sep(ctx); addMD5(ctx, hd.nonce); addMD5Sep(ctx); addMD5(ctx, outCNonce); - ctx.End(outSessionKey); - } - + ctx.End(outSessionKey); + } + ctx.End(outSessionKey); -}; - -/************************************************************/ -/* calculate request-digest/response-digest as per HTTP Digest spec */ +}; + +/************************************************************/ +/* calculate request-digest/response-digest as per HTTP Digest spec */ void httpDigestHandler::digestCalcResponse(const THttpAuthHeader& hd, const char* path, const char* method, const char* nonceCount, char* outResponse, char* outCNonce) { - char HA1[33]; - digestCalcHA1(hd, HA1, outCNonce); - - char HA2[33]; - MD5 ctx; - ctx.Init(); + char HA1[33]; + digestCalcHA1(hd, HA1, outCNonce); + + char HA2[33]; + MD5 ctx; + ctx.Init(); addMD5(ctx, method); addMD5Sep(ctx); addMD5(ctx, path); - //ignore auth-int - ctx.End(HA2); - - ctx.Init(); + //ignore auth-int + ctx.End(HA2); + + ctx.Init(); addMD5(ctx, HA1, 32); addMD5Sep(ctx); addMD5(ctx, Nonce_); addMD5Sep(ctx); - + if (hd.qop_auth) { - if (!*outCNonce) - generateCNonce(outCNonce); - + if (!*outCNonce) + generateCNonce(outCNonce); + addMD5(ctx, nonceCount, 8); addMD5Sep(ctx); addMD5(ctx, outCNonce); addMD5Sep(ctx); addMD5(ctx, "auth", 4); addMD5Sep(ctx); - } + } addMD5(ctx, HA2, 32); - ctx.End(outResponse); -} - -/************************************************************/ + ctx.End(outResponse); +} + +/************************************************************/ bool httpDigestHandler::processHeader(const THttpAuthHeader* header, const char* path, const char* method, const char* cnonce) { if (!User_ || !header || !header->use_auth || !header->realm || !header->nonce) - return false; - + return false; + if (Nonce_) { if (strcmp(Nonce_, header->nonce)) { free(Nonce_); Nonce_ = nullptr; NonceCount_ = 0; - } - } + } + } if (!Nonce_) { Nonce_ = strdup(header->nonce); NonceCount_ = 0; - } + } free(HeaderInstruction_); HeaderInstruction_ = nullptr; NonceCount_++; - - char nonceCount[20]; + + char nonceCount[20]; sprintf(nonceCount, "%08d", NonceCount_); - - char CNonce[50]; - if (cnonce) - strcpy(CNonce, cnonce); - else + + char CNonce[50]; + if (cnonce) + strcpy(CNonce, cnonce); + else CNonce[0] = 0; - - char response[33]; + + char response[33]; digestCalcResponse(*header, path, method, nonceCount, response, CNonce); - - //digest-response = 1#( username | realm | nonce | digest-uri - // | response | [ algorithm ] | [cnonce] | - // [opaque] | [message-qop] | - // [nonce-count] | [auth-param] ) - - TStringStream out; + + //digest-response = 1#( username | realm | nonce | digest-uri + // | response | [ algorithm ] | [cnonce] | + // [opaque] | [message-qop] | + // [nonce-count] | [auth-param] ) + + TStringStream out; out << WWW_PREFIX << "username=\"" << User_ << "\""; - out << ", realm=\"" << header->realm << "\""; - out << ", nonce=\"" << header->nonce << "\""; - out << ", uri=\"" << path << "\""; + out << ", realm=\"" << header->realm << "\""; + out << ", nonce=\"" << header->nonce << "\""; + out << ", uri=\"" << path << "\""; if (header->algorithm == 1) - out << ", algorithm=MD5-sess"; - else - out << ", algorithm=MD5"; - if (header->qop_auth) - out << ", qop=auth"; - out << ", nc=" << nonceCount; - if (CNonce[0]) - out << ", cnonce=\"" << CNonce << "\""; - out << ", response=\"" << response << "\""; - if (header->opaque) - out << ", opaque=\"" << header->opaque << "\""; - out << "\r\n"; - + out << ", algorithm=MD5-sess"; + else + out << ", algorithm=MD5"; + if (header->qop_auth) + out << ", qop=auth"; + out << ", nc=" << nonceCount; + if (CNonce[0]) + out << ", cnonce=\"" << CNonce << "\""; + out << ", response=\"" << response << "\""; + if (header->opaque) + out << ", opaque=\"" << header->opaque << "\""; + out << "\r\n"; + TString s_out = out.Str(); HeaderInstruction_ = strdup(s_out.c_str()); - - return true; -} - -/************************************************************/ -/************************************************************/ + + return true; +} + +/************************************************************/ +/************************************************************/ diff --git a/library/cpp/http/fetch/http_digest.h b/library/cpp/http/fetch/http_digest.h index 018107c2e4..3b1872d70b 100644 --- a/library/cpp/http/fetch/http_digest.h +++ b/library/cpp/http/fetch/http_digest.h @@ -1,10 +1,10 @@ #pragma once - + #include "httpheader.h" -#include <util/system/compat.h> +#include <util/system/compat.h> #include <library/cpp/http/misc/httpcodes.h> - + class httpDigestHandler { protected: const char* User_; @@ -12,36 +12,36 @@ protected: char* Nonce_; int NonceCount_; char* HeaderInstruction_; - - void clear(); - + + void clear(); + void generateCNonce(char* outCNonce); - + void digestCalcHA1(const THttpAuthHeader& hd, char* outSessionKey, char* outCNonce); - + void digestCalcResponse(const THttpAuthHeader& hd, const char* method, const char* path, const char* nonceCount, char* outResponse, char* outCNonce); - + public: - httpDigestHandler(); - ~httpDigestHandler(); - + httpDigestHandler(); + ~httpDigestHandler(); + void setAuthorization(const char* user, const char* password); bool processHeader(const THttpAuthHeader* header, const char* path, const char* method, const char* cnonce = nullptr); - + bool empty() const { return (!User_); - } - + } + const char* getHeaderInstruction() const; -}; +}; diff --git a/library/cpp/http/fetch/http_socket.cpp b/library/cpp/http/fetch/http_socket.cpp index 870d927489..1524ef04a8 100644 --- a/library/cpp/http/fetch/http_socket.cpp +++ b/library/cpp/http/fetch/http_socket.cpp @@ -1,30 +1,30 @@ -#include "httpload.h" -#include "http_digest.h" - -/************************************************************/ - -#ifdef USE_GNUTLS - -#include <gcrypt.h> -#include <gnutls/gnutls.h> +#include "httpload.h" +#include "http_digest.h" + +/************************************************************/ + +#ifdef USE_GNUTLS + +#include <gcrypt.h> +#include <gnutls/gnutls.h> #include <util/network/init.h> -#include <util/network/socket.h> -#include <util/system/mutex.h> - -/********************************************************/ -// HTTPS handler is used as implementation of -// socketAbstractHandler for work through HTTPS protocol - +#include <util/network/socket.h> +#include <util/system/mutex.h> + +/********************************************************/ +// HTTPS handler is used as implementation of +// socketAbstractHandler for work through HTTPS protocol + class socketSecureHandler: public socketRegularHandler { protected: bool IsValid_; gnutls_session Session_; gnutls_certificate_credentials Credits_; - + public: socketSecureHandler(); virtual ~socketSecureHandler(); - + virtual bool Good(); virtual int Connect(const TAddrList& addrs, TDuration Timeout); virtual void Disconnect(); @@ -32,175 +32,175 @@ public: virtual bool send(const char* message, ssize_t messlen); virtual bool peek(); virtual ssize_t read(void* buffer, ssize_t buflen); -}; - -/********************************************************/ -/********************************************************/ +}; + +/********************************************************/ +/********************************************************/ static int gcry_pthread_mutex_init(void** priv) { - int err = 0; - - try { + int err = 0; + + try { TMutex* lock = new TMutex; - *priv = lock; + *priv = lock; } catch (...) { - err = -1; - } - - return err; -} - + err = -1; + } + + return err; +} + static int gcry_pthread_mutex_destroy(void** lock) { delete static_cast<TMutex*>(*lock); - return 0; -} - + return 0; +} + static int gcry_pthread_mutex_lock(void** lock) { static_cast<TMutex*>(*lock)->Acquire(); - - return 0; -} - + + return 0; +} + static int gcry_pthread_mutex_unlock(void** lock) { static_cast<TMutex*>(*lock)->Release(); - - return 0; -} - -static struct gcry_thread_cbs gcry_threads_pthread = + + return 0; +} + +static struct gcry_thread_cbs gcry_threads_pthread = { GCRY_THREAD_OPTION_PTHREAD, NULL, gcry_pthread_mutex_init, gcry_pthread_mutex_destroy, gcry_pthread_mutex_lock, gcry_pthread_mutex_unlock, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}; - -/********************************************************/ + +/********************************************************/ struct https_initor { https_initor() { gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); - gnutls_global_init(); + gnutls_global_init(); InitNetworkSubSystem(); - } - + } + ~https_initor() { - gnutls_global_deinit(); - } -}; - -static https_initor _initor; - -/********************************************************/ + gnutls_global_deinit(); + } +}; + +static https_initor _initor; + +/********************************************************/ socketSecureHandler::socketSecureHandler() : socketRegularHandler() , IsValid_(false) , Session_() , Credits_() -{ -} - -/********************************************************/ +{ +} + +/********************************************************/ socketSecureHandler::~socketSecureHandler() { if (IsValid_) - Disconnect(); -} - -/********************************************************/ + Disconnect(); +} + +/********************************************************/ bool socketSecureHandler::Good() { return Socket_.Good() && IsValid_; -} - -/********************************************************/ +} + +/********************************************************/ int socketSecureHandler::Connect(const TAddrList& addrs, TDuration Timeout) { IsValid_ = false; - + int ret = socketRegularHandler::Connect(addrs, Timeout); - if (ret) - return ret; - + if (ret) + return ret; + gnutls_certificate_allocate_credentials(&Credits_); gnutls_init(&Session_, GNUTLS_CLIENT); gnutls_set_default_priority(Session_); gnutls_credentials_set(Session_, GNUTLS_CRD_CERTIFICATE, Credits_); - + SOCKET fd = Socket_; gnutls_transport_set_ptr(Session_, (gnutls_transport_ptr)fd); - + ret = gnutls_handshake(Session_); - + if (ret < 0) { - fprintf(stderr, "*** Handshake failed\n"); - gnutls_perror(ret); - + fprintf(stderr, "*** Handshake failed\n"); + gnutls_perror(ret); + gnutls_deinit(Session_); if (Credits_) { gnutls_certificate_free_credentials(Credits_); Credits_ = 0; - } - return 1; - } - + } + return 1; + } + IsValid_ = true; return !IsValid_; -} - -/********************************************************/ +} + +/********************************************************/ void socketSecureHandler::Disconnect() { if (IsValid_) { gnutls_bye(Session_, GNUTLS_SHUT_RDWR); IsValid_ = false; gnutls_deinit(Session_); - } - + } + if (Credits_) { gnutls_certificate_free_credentials(Credits_); Credits_ = 0; - } - - socketRegularHandler::Disconnect(); -} - -/********************************************************/ + } + + socketRegularHandler::Disconnect(); +} + +/********************************************************/ void socketSecureHandler::shutdown() { -} - -/********************************************************/ +} + +/********************************************************/ bool socketSecureHandler::send(const char* message, ssize_t messlen) { if (!IsValid_) - return false; + return false; ssize_t rv = gnutls_record_send(Session_, message, messlen); - return rv >= 0; -} - -/********************************************************/ + return rv >= 0; +} + +/********************************************************/ bool socketSecureHandler::peek() { - //ssize_t rv = gnutls_record_check_pending(mSession); - //return rv>0; - return true; -} - -/********************************************************/ + //ssize_t rv = gnutls_record_check_pending(mSession); + //return rv>0; + return true; +} + +/********************************************************/ ssize_t socketSecureHandler::read(void* buffer, ssize_t buflen) { if (!IsValid_) - return false; + return false; return gnutls_record_recv(Session_, (char*)buffer, buflen); -} - -#endif - -/************************************************************/ +} + +#endif + +/************************************************************/ socketAbstractHandler* socketHandlerFactory::chooseHandler(const THttpURL& url) { if (url.IsValidGlobal() && url.GetScheme() == THttpURL::SchemeHTTP) - return new socketRegularHandler; - + return new socketRegularHandler; + #ifdef USE_GNUTLS if (url.IsValidGlobal() && url.GetScheme() == THttpURL::SchemeHTTPS) - return new socketSecureHandler; + return new socketSecureHandler; #endif - + return nullptr; -} - -/************************************************************/ -socketHandlerFactory socketHandlerFactory::sInstance; -/************************************************************/ +} + +/************************************************************/ +socketHandlerFactory socketHandlerFactory::sInstance; +/************************************************************/ diff --git a/library/cpp/http/fetch/httpfetcher.h b/library/cpp/http/fetch/httpfetcher.h index 1c5b94a678..7fc251afd2 100644 --- a/library/cpp/http/fetch/httpfetcher.h +++ b/library/cpp/http/fetch/httpfetcher.h @@ -1,22 +1,22 @@ #pragma once - -#ifdef _MSC_VER + +#ifdef _MSC_VER #include <io.h> -#endif - +#endif + #include <library/cpp/http/misc/httpdate.h> - -#include "httpagent.h" -#include "httpparser.h" - -struct TFakeBackup { + +#include "httpagent.h" +#include "httpparser.h" + +struct TFakeBackup { int Write(void* /*buf*/, size_t /*size*/) { return 0; } -}; - -template <size_t bufsize = 5000> -struct TFakeAlloc { +}; + +template <size_t bufsize = 5000> +struct TFakeAlloc { void Shrink(void* /*buf*/, size_t /*size*/) { } void* Grab(size_t /*min*/, size_t* real) { @@ -24,17 +24,17 @@ struct TFakeAlloc { return buf; } char buf[bufsize]; -}; - +}; + template <typename TAlloc = TFakeAlloc<>, typename TCheck = TFakeCheck<>, - typename TWriter = TFakeBackup, + typename TWriter = TFakeBackup, typename TAgent = THttpAgent<>> class THttpFetcher: public THttpParser<TCheck>, public TAlloc, public TWriter, public TAgent { -public: - static const size_t TCP_MIN = 1500; - static int TerminateNow; - +public: + static const size_t TCP_MIN = 1500; + static int TerminateNow; + THttpFetcher() : THttpParser<TCheck>() , TAlloc() @@ -47,54 +47,54 @@ public: } int Fetch(THttpHeader* header, const char* path, const char* const* headers, int persistent, bool head_request = false) { - int ret = 0; - int fetcherr = 0; - - THttpParser<TCheck>::Init(header, head_request); + int ret = 0; + int fetcherr = 0; + + THttpParser<TCheck>::Init(header, head_request); const char* scheme = HttpUrlSchemeKindToString((THttpURL::TSchemeKind)TAgent::GetScheme()); size_t schemelen = strlen(scheme); - if (*path == '/') { + if (*path == '/') { header->base = TStringBuf(scheme, schemelen); header->base += TStringBuf("://", 3); header->base += TStringBuf(TAgent::pHostBeg, TAgent::pHostEnd - TAgent::pHostBeg); header->base += path; - } else { + } else { if (strlen(path) >= FETCHER_URL_MAX) { header->error = HTTP_URL_TOO_LARGE; return 0; } header->base = path; - } - - if ((ret = TAgent::RequestGet(path, headers, persistent, head_request))) { - header->error = (i16)ret; - return 0; - } - - bool inheader = 1; + } + + if ((ret = TAgent::RequestGet(path, headers, persistent, head_request))) { + header->error = (i16)ret; + return 0; + } + + bool inheader = 1; void *bufptr = nullptr, *buf = nullptr, *parsebuf = nullptr; - ssize_t got; - size_t buffree = 0, bufsize = 0, buflen = 0; - size_t maxsize = TCheck::GetMaxHeaderSize(); - do { - if (buffree < TCP_MIN) { - if (buf) { - TAlloc::Shrink(buf, buflen - buffree); - if (TWriter::Write(buf, buflen - buffree) < 0) { + ssize_t got; + size_t buffree = 0, bufsize = 0, buflen = 0; + size_t maxsize = TCheck::GetMaxHeaderSize(); + do { + if (buffree < TCP_MIN) { + if (buf) { + TAlloc::Shrink(buf, buflen - buffree); + if (TWriter::Write(buf, buflen - buffree) < 0) { buf = nullptr; - ret = EIO; - break; - } - } - if (!(buf = TAlloc::Grab(TCP_MIN, &buflen))) { - ret = ENOMEM; - break; - } - bufptr = buf; - buffree = buflen; - } - if ((got = TAgent::read(bufptr, buffree)) < 0) { - fetcherr = errno; + ret = EIO; + break; + } + } + if (!(buf = TAlloc::Grab(TCP_MIN, &buflen))) { + ret = ENOMEM; + break; + } + bufptr = buf; + buffree = buflen; + } + if ((got = TAgent::read(bufptr, buffree)) < 0) { + fetcherr = errno; if (errno == EINTR) header->error = HTTP_INTERRUPTED; else if (errno == ETIMEDOUT) @@ -102,43 +102,43 @@ public: else header->error = HTTP_CONNECTION_LOST; - break; - } - - parsebuf = bufptr; - bufptr = (char*)bufptr + got; - bufsize += got; - buffree -= got; - - THttpParser<TCheck>::Parse(parsebuf, got); - - if (header->error) + break; + } + + parsebuf = bufptr; + bufptr = (char*)bufptr + got; + bufsize += got; + buffree -= got; + + THttpParser<TCheck>::Parse(parsebuf, got); + + if (header->error) break; //if ANY error ocurred we will stop download that file or will have unprognosed stream position until MAX size reached - - if (inheader && THttpParser<TCheck>::GetState() != THttpParser<TCheck>::hp_in_header) { - inheader = 0; - if (TCheck::Check(header)) - break; - if (header->header_size > (long)maxsize) { - header->error = HTTP_HEADER_TOO_LARGE; - break; - } + + if (inheader && THttpParser<TCheck>::GetState() != THttpParser<TCheck>::hp_in_header) { + inheader = 0; + if (TCheck::Check(header)) + break; + if (header->header_size > (long)maxsize) { + header->error = HTTP_HEADER_TOO_LARGE; + break; + } } if (!inheader) { - maxsize = TCheck::GetMaxBodySize(header); - } - if (header->http_status >= HTTP_EXTENDED) - break; - if (bufsize > maxsize) { - header->error = inheader ? HTTP_HEADER_TOO_LARGE : HTTP_BODY_TOO_LARGE; - break; - } - if (TerminateNow) { - header->error = HTTP_INTERRUPTED; - break; - } - } while (THttpParser<TCheck>::GetState() > THttpParser<TCheck>::hp_eof); - + maxsize = TCheck::GetMaxBodySize(header); + } + if (header->http_status >= HTTP_EXTENDED) + break; + if (bufsize > maxsize) { + header->error = inheader ? HTTP_HEADER_TOO_LARGE : HTTP_BODY_TOO_LARGE; + break; + } + if (TerminateNow) { + header->error = HTTP_INTERRUPTED; + break; + } + } while (THttpParser<TCheck>::GetState() > THttpParser<TCheck>::hp_eof); + i64 Adjustment = 0; if (!header->error) { if (header->transfer_chunked) { @@ -150,22 +150,22 @@ public: Adjustment = 0; } - if (buf) { + if (buf) { TAlloc::Shrink(buf, buflen - buffree + Adjustment); - if (TWriter::Write(buf, buflen - buffree) < 0) - ret = EIO; - } - TCheck::CheckEndDoc(header); - if (ret || header->error || header->http_status >= HTTP_EXTENDED || header->connection_closed) { - TAgent::Disconnect(); - if (!fetcherr) - fetcherr = errno; - } - errno = fetcherr; - return ret; - } -}; - -template <typename TAlloc, typename TCheck, typename TWriter, typename TAgent> -int THttpFetcher<TAlloc, TCheck, TWriter, TAgent>::TerminateNow = 0; + if (TWriter::Write(buf, buflen - buffree) < 0) + ret = EIO; + } + TCheck::CheckEndDoc(header); + if (ret || header->error || header->http_status >= HTTP_EXTENDED || header->connection_closed) { + TAgent::Disconnect(); + if (!fetcherr) + fetcherr = errno; + } + errno = fetcherr; + return ret; + } +}; + +template <typename TAlloc, typename TCheck, typename TWriter, typename TAgent> +int THttpFetcher<TAlloc, TCheck, TWriter, TAgent>::TerminateNow = 0; diff --git a/library/cpp/http/fetch/httpfsm.h b/library/cpp/http/fetch/httpfsm.h index 62a27b6561..c4abdcd0d2 100644 --- a/library/cpp/http/fetch/httpfsm.h +++ b/library/cpp/http/fetch/httpfsm.h @@ -1,13 +1,13 @@ #pragma once - + #include "httpheader.h" -#include <util/system/maxlen.h> +#include <util/system/maxlen.h> #include <util/datetime/parser.h> - + #include <time.h> -struct THttpHeaderParser { +struct THttpHeaderParser { static constexpr int ErrFirstlineTypeMismatch = -3; static constexpr int ErrHeader = -2; static constexpr int Err = -1; @@ -16,34 +16,34 @@ struct THttpHeaderParser { static constexpr int Accepted = 2; int Execute(const void* inBuf, size_t len) { - return execute((unsigned char*)inBuf, (int)len); - } - + return execute((unsigned char*)inBuf, (int)len); + } + int Execute(TStringBuf str) { return Execute(str.data(), str.size()); } int Init(THttpHeader* h) { int ret = Init((THttpBaseHeader*)(h)); - hd = h; - hd->Init(); + hd = h; + hd->Init(); hreflangpos = hd->hreflangs; hreflangspace = HREFLANG_MAX; return ret; - } - + } + int Init(THttpAuthHeader* h) { - int ret = Init((THttpHeader*)(h)); - auth_hd = h; - return ret; - } + int ret = Init((THttpHeader*)(h)); + auth_hd = h; + return ret; + } int Init(THttpRequestHeader* h) { int ret = Init((THttpBaseHeader*)(h)); request_hd = h; request_hd->Init(); return ret; } - + THttpHeader* hd; long I; int Dc; @@ -51,7 +51,7 @@ struct THttpHeaderParser { char buf[FETCHER_URL_MAX]; size_t buflen; char* lastchar; - + const unsigned char* langstart; size_t langlen; @@ -62,10 +62,10 @@ struct THttpHeaderParser { THttpAuthHeader* auth_hd; THttpRequestHeader* request_hd; - -private: + +private: THttpBaseHeader* base_hd; - int cs; + int cs; private: int Init(THttpBaseHeader* header) { @@ -78,27 +78,27 @@ private: } int execute(unsigned char* inBuf, int len); - void init(); -}; - -struct THttpChunkParser { + void init(); +}; + +struct THttpChunkParser { int Execute(const void* inBuf, int len) { - return execute((unsigned char*)inBuf, len); - } - - int Init() { - init(); - return 0; - } - + return execute((unsigned char*)inBuf, len); + } + + int Init() { + init(); + return 0; + } + int chunk_length; char* lastchar; long I; int Dc; i64 cnt64; - -private: - int cs; + +private: + int cs; int execute(unsigned char* inBuf, int len); - void init(); -}; + void init(); +}; diff --git a/library/cpp/http/fetch/httpfsm.rl6 b/library/cpp/http/fetch/httpfsm.rl6 index 83557b144e..eab0328b18 100644 --- a/library/cpp/http/fetch/httpfsm.rl6 +++ b/library/cpp/http/fetch/httpfsm.rl6 @@ -1,70 +1,70 @@ -#include <stdio.h> -#include <time.h> - +#include <stdio.h> +#include <time.h> + #include <library/cpp/charset/doccodes.h> #include <library/cpp/charset/codepage.h> #include <library/cpp/http/misc/httpcodes.h> #include <util/datetime/base.h> #include <util/generic/ylimits.h> #include <algorithm> // max - + #include <library/cpp/http/fetch/httpheader.h> #include <library/cpp/http/fetch/httpfsm.h> - + #ifdef _MSC_VER #pragma warning(disable: 4702) // unreachable code #endif #define c(i) I = i; #define m(i) I = std::max(I, (long)i); - -static inline int X(unsigned char c) { - return (c >= 'A' ? ((c & 0xdf) - 'A' + 10) : (c - '0')); -} - -template <typename x> -static inline void guard(x &val) { - val = (val >= -1) ? -4 - val : -2; // f(-2) = -2 -} - -template <typename x> -static inline void setguarded(x &val, long cnt) { - val = (val == -4 - -1 || cnt == -4 -val) ? cnt : -2; -} - -//////////////////////////////////////////////////////////////////// -/// HTTP PARSER -//////////////////////////////////////////////////////////////////// - -%%{ -machine http_header_parser; - + +static inline int X(unsigned char c) { + return (c >= 'A' ? ((c & 0xdf) - 'A' + 10) : (c - '0')); +} + +template <typename x> +static inline void guard(x &val) { + val = (val >= -1) ? -4 - val : -2; // f(-2) = -2 +} + +template <typename x> +static inline void setguarded(x &val, long cnt) { + val = (val == -4 - -1 || cnt == -4 -val) ? cnt : -2; +} + +//////////////////////////////////////////////////////////////////// +/// HTTP PARSER +//////////////////////////////////////////////////////////////////// + +%%{ +machine http_header_parser; + include HttpDateTimeParser "../../../../util/datetime/parser.rl6"; -alphtype unsigned char; - -################# 2.2 Basic Rules ################# -eol = '\r'? '\n'; -ws = [ \t]; -lw = '\r'? '\n'? ws; +alphtype unsigned char; + +################# 2.2 Basic Rules ################# +eol = '\r'? '\n'; +ws = [ \t]; +lw = '\r'? '\n'? ws; separator = [()<>@,;:\\"/\[\]?={}]; -token_char = [!-~] - separator; # http tokens chars -url_char = [!-~] - ["<>\[\]\\^`{}|]; # uric chars -text_char = ws | 33..126 | 128..255; -any_text_char = any - [\r\n]; - -lws = lw*; -eoh = lws eol; -token = token_char+; -ex_token = (token_char | ws)* token_char; -text = (text_char | lw)*; -any_text = (any_text_char | lw)*; -def = lws ':' lws; - +token_char = [!-~] - separator; # http tokens chars +url_char = [!-~] - ["<>\[\]\\^`{}|]; # uric chars +text_char = ws | 33..126 | 128..255; +any_text_char = any - [\r\n]; + +lws = lw*; +eoh = lws eol; +token = token_char+; +ex_token = (token_char | ws)* token_char; +text = (text_char | lw)*; +any_text = (any_text_char | lw)*; +def = lws ':' lws; + action clear_buf { buflen = 0; } action update_buf { if (buflen < sizeof(buf)) buf[buflen++] = fc; } - -################################################### + +################################################### ############ response status line ################# action set_minor { base_hd->http_minor = I; } action set_status { @@ -75,14 +75,14 @@ action set_status { return -3; } } - + status_code = int3; http_major = int; http_minor = int; reason_phrase = ws+ text_char*; http_version = "http/"i http_major '.' http_minor %set_minor; response_status_line = http_version ws+ status_code reason_phrase? eol %set_status; - + ############ request status line ################# action set_request_uri { if (request_hd && buflen < FETCHER_URL_MAX) { @@ -116,44 +116,44 @@ request_uri = (token_char | separator)+ >clear_buf $update_buf %set_request_uri; request_status_line = http_method ws+ request_uri ws+ http_version eoh; -################# connection ###################### +################# connection ###################### action beg_connection { guard(base_hd->connection_closed); I = -1; } action set_connection { setguarded(base_hd->connection_closed, I); } - -c_token = "close"i %{m(1)} - | "keep-alive"i %{m(0)}; -c_tokenlist = c_token (lws ',' lws c_token)?; + +c_token = "close"i %{m(1)} + | "keep-alive"i %{m(0)}; +c_tokenlist = c_token (lws ',' lws c_token)?; connection = "connection"i def %beg_connection c_tokenlist eoh %set_connection; - -################# content-encoding ################ + +################# content-encoding ################ action beg_content_encoding { I = HTTP_COMPRESSION_ERROR; } action set_content_encoding { base_hd->compression_method = ((base_hd->compression_method == HTTP_COMPRESSION_UNSET || base_hd->compression_method == I) ? I : (int)HTTP_COMPRESSION_ERROR); } - -ce_tokenlist = "identity"i %{c(HTTP_COMPRESSION_IDENTITY)} - | "gzip"i %{c(HTTP_COMPRESSION_GZIP)} - | "x-gzip"i %{c(HTTP_COMPRESSION_GZIP)} - | "deflate"i %{c(HTTP_COMPRESSION_DEFLATE)} - | "compress"i %{c(HTTP_COMPRESSION_COMPRESS)} - | "x-compress"i %{c(HTTP_COMPRESSION_COMPRESS)}; + +ce_tokenlist = "identity"i %{c(HTTP_COMPRESSION_IDENTITY)} + | "gzip"i %{c(HTTP_COMPRESSION_GZIP)} + | "x-gzip"i %{c(HTTP_COMPRESSION_GZIP)} + | "deflate"i %{c(HTTP_COMPRESSION_DEFLATE)} + | "compress"i %{c(HTTP_COMPRESSION_COMPRESS)} + | "x-compress"i %{c(HTTP_COMPRESSION_COMPRESS)}; content_encoding = "content-encoding"i def %beg_content_encoding ce_tokenlist eoh %set_content_encoding; - -################# transfer-encoding ############### + +################# transfer-encoding ############### action beg_encoding { guard(base_hd->transfer_chunked); } action set_encoding { setguarded(base_hd->transfer_chunked, I); } - -e_tokenlist = "identity"i %{c(0)} - | "chunked"i %{c(1)}; + +e_tokenlist = "identity"i %{c(0)} + | "chunked"i %{c(1)}; transfer_encoding = "transfer-encoding"i def %beg_encoding e_tokenlist eoh %set_encoding; - -################# content-length ################## + +################# content-length ################## action beg_content_length { guard(base_hd->content_length); } action set_content_length { setguarded(base_hd->content_length, I); } - + content_length = "content-length"i def %beg_content_length int eoh %set_content_length; - + ################# content-range ################### action beg_content_range_start { guard(base_hd->content_range_start); I = -1; } action set_content_range_start { setguarded(base_hd->content_range_start, I); } @@ -166,7 +166,7 @@ content_range = "content-range"i def "bytes"i sp %beg_content_range_start int %beg_content_range_end int '/' %set_content_range_end %beg_content_range_el int eoh %set_content_range_el; -################# accept-ranges ################### +################# accept-ranges ################### action beg_accept_ranges { if (hd) { guard(hd->accept_ranges); @@ -174,21 +174,21 @@ action beg_accept_ranges { } } action set_accept_ranges { if (hd) setguarded(hd->accept_ranges, I); } - -ar_tokenlist = "bytes"i %{c(1)} - | "none"i %{c(0)}; + +ar_tokenlist = "bytes"i %{c(1)} + | "none"i %{c(0)}; accept_ranges = "accept-ranges"i def %beg_accept_ranges ar_tokenlist eoh %set_accept_ranges; - -################# content-type #################### + +################# content-type #################### action beg_mime { guard(base_hd->mime_type); } action set_mime { setguarded(base_hd->mime_type, I); } action set_charset { if (buflen < FETCHER_URL_MAX) { - buf[buflen++] = 0; + buf[buflen++] = 0; base_hd->charset = EncodingHintByName((const char*)buf); - } -} - + } +} + mime_type = "text/plain"i %{c(MIME_TEXT)} | "text/html"i %{c(MIME_HTML)} | "application/pdf"i %{c(MIME_PDF)} @@ -234,36 +234,36 @@ mime_type = "text/plain"i %{c(MIME_TEXT)} charset_name = token_char+ >clear_buf $update_buf; mime_param = "charset"i ws* '=' ws* '"'? charset_name '"'? %set_charset @2 - | token ws* '=' ws* '"'? token '"'? @1 - | text $0; -mime_parms = (lws ';' lws mime_param)*; + | token ws* '=' ws* '"'? token '"'? @1 + | text $0; +mime_parms = (lws ';' lws mime_param)*; content_type = "content-type"i def %beg_mime mime_type mime_parms eoh %set_mime; - -################# last modified ################### + +################# last modified ################### action beg_modtime { guard(base_hd->http_time); } action set_modtime { setguarded(base_hd->http_time, DateTimeFields.ToTimeT(-1)); } - + last_modified = "last-modified"i def %beg_modtime http_date eoh %set_modtime; - -################# location ######################## + +################# location ######################## action set_location { while (buflen > 0 && (buf[buflen - 1] == ' ' || buf[buflen - 1] == '\t')) { buflen --; } if (hd && buflen < FETCHER_URL_MAX) { hd->location = TStringBuf(buf, buflen); - } -} - + } +} + action set_status_303{ if (hd) hd->http_status = 303; } - + url = url_char+ >clear_buf $update_buf; loc_url = any_text_char+ >clear_buf $update_buf; location = "location"i def loc_url eoh %set_location; refresh = "refresh"i def int ';' lws "url="i loc_url eoh %set_location; - + ################# x-robots-tag ################ action set_x_robots { if (hd && AcceptingXRobots) { @@ -349,56 +349,56 @@ action set_squid_error { squid_error = "X-Yandex-Squid-Error"i def any_text eoh %set_squid_error; -################# auth ######################## +################# auth ######################## action init_auth { - if (auth_hd) - auth_hd->use_auth=true; -} - + if (auth_hd) + auth_hd->use_auth=true; +} + action update_auth_buf - { if (auth_hd && buflen < sizeof(buf)) buf[buflen++] = *fpc; } - -quoted_str = /"/ (text_char - /"/)* /"/ >2; + { if (auth_hd && buflen < sizeof(buf)) buf[buflen++] = *fpc; } + +quoted_str = /"/ (text_char - /"/)* /"/ >2; auth_quoted_str = ( /"/ ( ( text_char - /"/ )* >clear_buf $update_auth_buf ) /"/ ) > 2; - -# do not support auth-int, too heavy procedure - -qop_auth_option = "auth"i @1 %{if(auth_hd) auth_hd->qop_auth = true; }; - -qop_option = ( qop_auth_option @1 ) | (( token-"auth"i) $0 ); - -auth_good_param = ( "nonce"i /=/ auth_quoted_str ) + +# do not support auth-int, too heavy procedure + +qop_auth_option = "auth"i @1 %{if(auth_hd) auth_hd->qop_auth = true; }; + +qop_option = ( qop_auth_option @1 ) | (( token-"auth"i) $0 ); + +auth_good_param = ( "nonce"i /=/ auth_quoted_str ) %{if (auth_hd && buflen < FETCHER_URL_MAX-1) { - buf[buflen++] = 0; - auth_hd->nonce = strdup((const char*)buf); - }} - | ( "realm"i /=/ auth_quoted_str ) + buf[buflen++] = 0; + auth_hd->nonce = strdup((const char*)buf); + }} + | ( "realm"i /=/ auth_quoted_str ) %{if (auth_hd && buflen < FETCHER_URL_MAX-1) { - buf[buflen++] = 0; - auth_hd->realm = strdup((const char*)buf); - }} - | ( "opaque"i /=/ auth_quoted_str ) + buf[buflen++] = 0; + auth_hd->realm = strdup((const char*)buf); + }} + | ( "opaque"i /=/ auth_quoted_str ) %{if (auth_hd && buflen < FETCHER_URL_MAX-1) { - buf[buflen++] = 0; - auth_hd->opaque = strdup((const char*)buf); - }} - | "stale"i /=/ "true"i - %{if (auth_hd) auth_hd->stale = true; } - | "algorithm"i /=/ "md5"i /-/ "sess"i - %{if (auth_hd) auth_hd->algorithm = 1; } - | ( "qop"i /="/ qop_option (ws* "," ws* qop_option)* /"/); - -auth_param = auth_good_param @1 | - ( (token - ( "nonce"i | "opaque"i | "realm"i | "qop"i ) ) - /=/ (token | quoted_str ) ) $0; - -auth_params = auth_param ( ws* /,/ ws* auth_param )*; - + buf[buflen++] = 0; + auth_hd->opaque = strdup((const char*)buf); + }} + | "stale"i /=/ "true"i + %{if (auth_hd) auth_hd->stale = true; } + | "algorithm"i /=/ "md5"i /-/ "sess"i + %{if (auth_hd) auth_hd->algorithm = 1; } + | ( "qop"i /="/ qop_option (ws* "," ws* qop_option)* /"/); + +auth_param = auth_good_param @1 | + ( (token - ( "nonce"i | "opaque"i | "realm"i | "qop"i ) ) + /=/ (token | quoted_str ) ) $0; + +auth_params = auth_param ( ws* /,/ ws* auth_param )*; + digest_challenge = ("digest"i %init_auth ws+ auth_params) | - ((token-"digest"i) text); - -auth = "www-authenticate"i def digest_challenge eoh; - + ((token-"digest"i) text); + +auth = "www-authenticate"i def digest_challenge eoh; + ###################### host ####################### action set_host { if (request_hd && buflen < HOST_MAX) { @@ -562,8 +562,8 @@ action set_request_priority { request_priority = "x-yandex-request-priority"i def int eoh %set_request_priority; -################# message header ################## -other_header = ( ex_token - "www-authenticate"i ) def any_text eoh; +################# message header ################## +other_header = ( ex_token - "www-authenticate"i ) def any_text eoh; message_header = other_header $0 | connection @1 | content_encoding @1 @@ -595,90 +595,90 @@ request_header = message_header $0 | request_cache_control @1 | response_timeout @1 | request_priority @1; - -################# main ############################ + +################# main ############################ action accepted { lastchar = (char*)fpc; return 2; } - + main := ((response_status_line ('\r'? response_header)*) | (request_status_line ('\r' ? request_header)*)) eol @accepted; - -}%% - -%% write data; - -int THttpHeaderParser::execute(unsigned char *inBuf, int len) { - const unsigned char *p = inBuf; - const unsigned char *pe = p + len; - %% write exec; - if (cs == http_header_parser_error) - return -1; - else if (cs == http_header_parser_first_final) - return 0; - else - return 1; -} - -void THttpHeaderParser::init() { - %% write init; -} - -%%{ -machine http_chunk_parser; - -alphtype unsigned char; - + +}%% + +%% write data; + +int THttpHeaderParser::execute(unsigned char *inBuf, int len) { + const unsigned char *p = inBuf; + const unsigned char *pe = p + len; + %% write exec; + if (cs == http_header_parser_error) + return -1; + else if (cs == http_header_parser_first_final) + return 0; + else + return 1; +} + +void THttpHeaderParser::init() { + %% write init; +} + +%%{ +machine http_chunk_parser; + +alphtype unsigned char; + action clear_hex { cnt64 = 0; } action update_hex { cnt64 = 16 * cnt64 + X(fc); if(cnt64 > Max<int>()) return -2; } action set_chunk { chunk_length = static_cast<int>(cnt64); } action accepted { lastchar = (char*)fpc; return 2; } - -eol = '\r'? '\n'; -ws = [ \t]; -sp = ' '; -lw = '\r'? '\n'? ws; -separator = [()<>@,;:\\"/\[\]?={}]; -token_char = [!-~] - separator; # http tokens chars -url_char = [!-~] - ["<>\[\]\\^`{}|]; # uric chars -text_char = ws | 33..127 | 160..255; - -lws = lw*; -eoh = lws eol; -token = token_char+; -text = (text_char | lw)*; -def = lws ':' lws; - + +eol = '\r'? '\n'; +ws = [ \t]; +sp = ' '; +lw = '\r'? '\n'? ws; +separator = [()<>@,;:\\"/\[\]?={}]; +token_char = [!-~] - separator; # http tokens chars +url_char = [!-~] - ["<>\[\]\\^`{}|]; # uric chars +text_char = ws | 33..127 | 160..255; + +lws = lw*; +eoh = lws eol; +token = token_char+; +text = (text_char | lw)*; +def = lws ':' lws; + hex = (xdigit+) >clear_hex $update_hex; -quoted_string = '"' ((text_char - '"') $0 | '\\"' @1)* '"'; - -chunk_ext_val = token | quoted_string; -chunk_ext_name = token; -chunk_extension = ws* (';' chunk_ext_name ws* '=' ws* chunk_ext_val ws*)*; - -entity_header = token def text eoh; -trailer = entity_header*; - +quoted_string = '"' ((text_char - '"') $0 | '\\"' @1)* '"'; + +chunk_ext_val = token | quoted_string; +chunk_ext_name = token; +chunk_extension = ws* (';' chunk_ext_name ws* '=' ws* chunk_ext_val ws*)*; + +entity_header = token def text eoh; +trailer = entity_header*; + chunk = (hex - '0'+) chunk_extension? %set_chunk; -last_chunk = '0'+ chunk_extension? eol trailer; +last_chunk = '0'+ chunk_extension? eol trailer; main := eol (chunk $0 | last_chunk @1) eol @accepted; - -}%% - -%% write data; - -int THttpChunkParser::execute(unsigned char *inBuf, int len) { - const unsigned char *p = inBuf; - const unsigned char *pe = p + len; - %% write exec; - if (cs == http_chunk_parser_error) - return -1; - else if (cs == http_chunk_parser_first_final) - return 0; - else - return 1; -} - -void THttpChunkParser::init() { - chunk_length = 0; - %% write init; -} + +}%% + +%% write data; + +int THttpChunkParser::execute(unsigned char *inBuf, int len) { + const unsigned char *p = inBuf; + const unsigned char *pe = p + len; + %% write exec; + if (cs == http_chunk_parser_error) + return -1; + else if (cs == http_chunk_parser_first_final) + return 0; + else + return 1; +} + +void THttpChunkParser::init() { + chunk_length = 0; + %% write init; +} diff --git a/library/cpp/http/fetch/httpheader.h b/library/cpp/http/fetch/httpheader.h index 20f8e0956b..b2810bbd41 100644 --- a/library/cpp/http/fetch/httpheader.h +++ b/library/cpp/http/fetch/httpheader.h @@ -1,5 +1,5 @@ #pragma once - + #include "exthttpcodes.h" #include <library/cpp/mime/types/mime.h> @@ -11,11 +11,11 @@ #include <util/system/maxlen.h> #include <ctime> -#include <cstdio> -#include <cstdlib> -#include <cstring> +#include <cstdio> +#include <cstdlib> +#include <cstring> #include <algorithm> - + // This is ugly solution but here a lot of work to do it the right way. #define FETCHER_URL_MAX 8192 @@ -29,16 +29,16 @@ extern const i32 DEFAULT_RESPONSE_TIMEOUT; /// == -1 #define MAX_LANGREGION_LEN 4 #define MAXWORD_LEN 55 -enum HTTP_COMPRESSION { +enum HTTP_COMPRESSION { HTTP_COMPRESSION_UNSET = 0, HTTP_COMPRESSION_ERROR = 1, - HTTP_COMPRESSION_IDENTITY = 2, + HTTP_COMPRESSION_IDENTITY = 2, HTTP_COMPRESSION_GZIP = 3, HTTP_COMPRESSION_DEFLATE = 4, - HTTP_COMPRESSION_COMPRESS = 5, + HTTP_COMPRESSION_COMPRESS = 5, HTTP_COMPRESSION_MAX = 6 -}; - +}; + enum HTTP_METHOD { HTTP_METHOD_UNDEFINED = -1, HTTP_METHOD_OPTIONS, @@ -78,25 +78,25 @@ public: TString base; public: - void Init() { + void Init() { error = 0; - header_size = 0; - entity_size = 0; - content_length = -1; - http_time = -1; - http_minor = -1; - mime_type = -1; - charset = -1; - compression_method = HTTP_COMPRESSION_UNSET; - transfer_chunked = -1; + header_size = 0; + entity_size = 0; + content_length = -1; + http_time = -1; + http_minor = -1; + mime_type = -1; + charset = -1; + compression_method = HTTP_COMPRESSION_UNSET; + transfer_chunked = -1; connection_closed = HTTP_CONNECTION_UNDEFINED; content_range_start = -1; content_range_end = -1; content_range_entity_length = -1; base.clear(); - } - - void Print() const { + } + + void Print() const { printf("content_length: %" PRIi64 "\n", content_length); printf("http_time: %" PRIi64 "\n", http_time); printf("http_minor: %" PRIi8 "\n", http_minor); @@ -110,22 +110,22 @@ public: printf("content_range_entity_length: %" PRIi64 "\n", content_range_entity_length); printf("base: \"%s\"\n", base.c_str()); printf("error: %" PRIi16 "\n", error); - } - - int SetBase(const char* path, + } + + int SetBase(const char* path, const char* hostNamePtr = nullptr, int hostNameLength = 0) { if (*path == '/') { base = "http://"; base += TStringBuf(hostNamePtr, hostNameLength); base += path; - } else { + } else { base = path; - } - return error; - } -}; - + } + return error; + } +}; + enum { HREFLANG_MAX = FETCHER_URL_MAX * 2 }; /// Class represents Http Response Header. struct THttpHeader: public THttpBaseHeader { @@ -165,7 +165,7 @@ public: }; struct THttpRequestHeader: public THttpBaseHeader { -public: +public: TString request_uri; char host[HOST_MAX]; char from[MAXWORD_LEN]; @@ -184,7 +184,7 @@ public: THttpRequestHeader() { Init(); } - + void Init() { request_uri.clear(); host[0] = 0; @@ -201,7 +201,7 @@ public: if_modified_since = DEFAULT_IF_MODIFIED_SINCE; THttpBaseHeader::Init(); } - + void Print() const { THttpBaseHeader::Print(); printf("request_uri: \"%s\"\n", request_uri.c_str()); @@ -213,7 +213,7 @@ public: printf("max_age: %" PRIi32 "\n", max_age); printf("if_modified_since: %" PRIi64 "\n", if_modified_since); } - + /// It doesn't care about errors in request or headers, where /// request_uri equals to '*'. /// This returns copy of the string, which you have to delete. @@ -225,20 +225,20 @@ public: url = HTTP_PREFIX; url += host; url += request_uri; - } + } return url; } - + char* GetUrl(char* buffer, size_t size) { if (host[0] == 0 || !strcmp(host, "")) { strlcpy(buffer, request_uri.c_str(), size); } else { snprintf(buffer, size, "http://%s%s", host, request_uri.c_str()); - } + } return buffer; } }; - + class THttpAuthHeader: public THttpHeader { public: char* realm; @@ -282,6 +282,6 @@ public: printf("stale: %d\n", stale); printf("algorithm: %d\n", algorithm); printf("qop_auth: %d\n", qop_auth); - } + } } -}; +}; diff --git a/library/cpp/http/fetch/httpload.cpp b/library/cpp/http/fetch/httpload.cpp index f944d7906a..82ea8900b5 100644 --- a/library/cpp/http/fetch/httpload.cpp +++ b/library/cpp/http/fetch/httpload.cpp @@ -1,7 +1,7 @@ -#include "httpload.h" - -/************************************************************/ -/************************************************************/ +#include "httpload.h" + +/************************************************************/ +/************************************************************/ httpAgentReader::httpAgentReader(httpSpecialAgent& agent, const char* baseUrl, bool assumeConnectionClosed, @@ -13,109 +13,109 @@ httpAgentReader::httpAgentReader(httpSpecialAgent& agent, , BufPtr_(Buffer_) , BufSize_(bufSize) , BufRest_(0) -{ - HeadRequest = false; +{ + HeadRequest = false; Header = &Header_; - if (use_auth) + if (use_auth) HeaderParser.Init(&Header_); - else - HeaderParser.Init(Header); + else + HeaderParser.Init(Header); setAssumeConnectionClosed(assumeConnectionClosed ? 1 : 0); Header_.SetBase(baseUrl); - + if (Header_.error) - State = hp_error; - else - State = hp_in_header; -} - -/************************************************************/ + State = hp_error; + else + State = hp_in_header; +} + +/************************************************************/ httpAgentReader::~httpAgentReader() { delete[] Buffer_; -} - -/************************************************************/ +} + +/************************************************************/ void httpAgentReader::readBuf() { assert(BufRest_ == 0); if (!BufPtr_) { BufRest_ = -1; - return; - } - + return; + } + BufRest_ = Agent_.read(Buffer_, BufSize_); if (BufRest_ <= 0) { BufRest_ = -1; BufPtr_ = nullptr; - } else { + } else { BufPtr_ = Buffer_; - //cout << "BUF: " << mBuffer << endl << endl; - } -} - -/************************************************************/ + //cout << "BUF: " << mBuffer << endl << endl; + } +} + +/************************************************************/ const THttpHeader* httpAgentReader::readHeader() { while (State == hp_in_header) { if (!step()) { Header_.error = HTTP_CONNECTION_LOST; return nullptr; - } + } ParseGeneric(BufPtr_, BufRest_); - } + } if (State == hp_eof || State == hp_error) { BufPtr_ = nullptr; BufRest_ = -1; - } + } if (State == hp_error || Header_.error) return nullptr; return &Header_; -} - -/************************************************************/ +} + +/************************************************************/ long httpAgentReader::readPortion(void*& buf) { assert(State != hp_in_header); - - long Chunk = 0; + + long Chunk = 0; do { if (BufSize_ == 0 && !BufPtr_) - return 0; - - if (!step()) - return 0; - + return 0; + + if (!step()) + return 0; + Chunk = ParseGeneric(BufPtr_, BufRest_); buf = BufPtr_; - + if (State == hp_error && Header_.entity_size > Header_.content_length) { Chunk -= (Header_.entity_size - Header_.content_length); BufPtr_ = (char*)BufPtr_ + Chunk; BufRest_ = 0; State = hp_eof; Header_.error = 0; - break; - } - + break; + } + BufPtr_ = (char*)BufPtr_ + Chunk; BufRest_ -= Chunk; - + if (State == hp_eof || State == hp_error) { BufRest_ = -1; BufPtr_ = nullptr; - } - } while (!Chunk); - return Chunk; -} - -/************************************************************/ + } + } while (!Chunk); + return Chunk; +} + +/************************************************************/ bool httpAgentReader::skipTheRest() { void* b; - while (!eof()) - readPortion(b); + while (!eof()) + readPortion(b); return (State == hp_eof); -} - -/************************************************************/ -/************************************************************/ +} + +/************************************************************/ +/************************************************************/ httpLoadAgent::httpLoadAgent(bool handleAuthorization, socketHandlerFactory& factory) : Factory_(factory) @@ -126,82 +126,82 @@ httpLoadAgent::httpLoadAgent(bool handleAuthorization, , Headers_() , ErrCode_(0) , RealHost_(nullptr) -{ -} - -/************************************************************/ +{ +} + +/************************************************************/ httpLoadAgent::~httpLoadAgent() { delete Reader_; free(RealHost_); -} - -/************************************************************/ +} + +/************************************************************/ void httpLoadAgent::clearReader() { if (Reader_) { - bool opened = false; + bool opened = false; if (PersistentConn_) { const THttpHeader* H = Reader_->readHeader(); if (H && !H->connection_closed) { Reader_->skipTheRest(); - opened = true; - } - } - if (!opened) - Disconnect(); + opened = true; + } + } + if (!opened) + Disconnect(); delete Reader_; Reader_ = nullptr; - } + } ErrCode_ = 0; -} -/************************************************************/ +} +/************************************************************/ void httpLoadAgent::setRealHost(const char* hostname) { free(RealHost_); - if (hostname) + if (hostname) RealHost_ = strdup(hostname); - else + else RealHost_ = nullptr; ErrCode_ = 0; -} - -/************************************************************/ +} + +/************************************************************/ void httpLoadAgent::setIMS(const char* ifModifiedSince) { - char ims_buf[100]; - snprintf(ims_buf, 100, "If-Modified-Since: %s\r\n", - ifModifiedSince); + char ims_buf[100]; + snprintf(ims_buf, 100, "If-Modified-Since: %s\r\n", + ifModifiedSince); Headers_.push_back(ims_buf); -} - -/************************************************************/ +} + +/************************************************************/ void httpLoadAgent::addHeaderInstruction(const char* instr) { Headers_.push_back(instr); -} - -/************************************************************/ +} + +/************************************************************/ void httpLoadAgent::dropHeaderInstructions() { Headers_.clear(); -} - -/************************************************************/ +} + +/************************************************************/ bool httpLoadAgent::startRequest(const THttpURL& url, bool persistent, const TAddrList& addrs) -{ - clearReader(); +{ + clearReader(); ErrCode_ = 0; - + URL_.Clear(); URL_ = url; PersistentConn_ = persistent; if (!URL_.IsValidAbs()) - return false; + return false; if (!HandleAuthorization_ && !URL_.IsNull(THttpURL::FlagAuth)) - return false; - + return false; + return doSetHost(addrs) && doStartRequest(); -} - -/************************************************************/ +} + +/************************************************************/ bool httpLoadAgent::startRequest(const char* url, const char* url_to_merge, bool persistent, @@ -226,148 +226,148 @@ bool httpLoadAgent::startRequest(const char* url, const char* url_to_merge, bool persistent, ui32 ip) { - clearReader(); - + clearReader(); + URL_.Clear(); PersistentConn_ = persistent; - - long flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet; + + long flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet; if (HandleAuthorization_) - flags |= THttpURL::FeatureAuthSupported; - + flags |= THttpURL::FeatureAuthSupported; + if (URL_.Parse(url, flags, url_to_merge) || !URL_.IsValidGlobal()) - return false; - + return false; + return doSetHost(TAddrList::MakeV4Addr(ip, URL_.GetPort())) && doStartRequest(); -} - -/************************************************************/ +} + +/************************************************************/ bool httpLoadAgent::doSetHost(const TAddrList& addrs) { socketAbstractHandler* h = Factory_.chooseHandler(URL_); - if (!h) - return false; - Socket.setHandler(h); - + if (!h) + return false; + Socket.setHandler(h); + if (addrs.size()) { ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost), URL_.GetPort(), addrs); - } else { + } else { ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost), URL_.GetPort()); - } + } if (ErrCode_) - return false; - + return false; + if (RealHost_) { - free(Hostheader); + free(Hostheader); Hostheader = (char*)malloc(strlen(RealHost_) + 20); sprintf(Hostheader, "Host: %s\r\n", RealHost_); - } - + } + if (!URL_.IsNull(THttpURL::FlagAuth)) { if (!HandleAuthorization_) { ErrCode_ = HTTP_UNAUTHORIZED; - return false; - } - + return false; + } + Digest_.setAuthorization(URL_.Get(THttpURL::FieldUsername), URL_.Get(THttpURL::FieldPassword)); - } - - return true; -} - -/************************************************************/ + } + + return true; +} + +/************************************************************/ bool httpLoadAgent::setHost(const char* host_url, const TAddrList& addrs) { - clearReader(); - + clearReader(); + URL_.Clear(); PersistentConn_ = true; - - long flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet; + + long flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet; if (HandleAuthorization_) - flags |= THttpURL::FeatureAuthSupported; - + flags |= THttpURL::FeatureAuthSupported; + if (URL_.Parse(host_url, flags) || !URL_.IsValidGlobal()) - return false; - + return false; + return doSetHost(addrs); -} - -/************************************************************/ +} + +/************************************************************/ bool httpLoadAgent::startOneRequest(const char* local_url) { - clearReader(); - - THttpURL lURL; + clearReader(); + + THttpURL lURL; if (lURL.Parse(local_url, THttpURL::FeaturesNormalizeSet) || lURL.IsValidGlobal()) - return false; - + return false; + URL_.SetInMemory(THttpURL::FieldPath, lURL.Get(THttpURL::FieldPath)); URL_.SetInMemory(THttpURL::FieldQuery, lURL.Get(THttpURL::FieldQuery)); URL_.Rewrite(); - - return doStartRequest(); -} - -/************************************************************/ + + return doStartRequest(); +} + +/************************************************************/ bool httpLoadAgent::doStartRequest() { TString urlStr = URL_.PrintS(THttpURL::FlagPath | THttpURL::FlagQuery); - if (!urlStr) - urlStr = "/"; - + if (!urlStr) + urlStr = "/"; + for (int step = 0; step < 10; step++) { const char* digestHeader = Digest_.getHeaderInstruction(); - + unsigned i = (digestHeader) ? 2 : 1; - const char** headers = + const char** headers = (const char**)(alloca((i + Headers_.size()) * sizeof(char*))); - + for (i = 0; i < Headers_.size(); i++) headers[i] = Headers_[i].c_str(); - if (digestHeader) - headers[i++] = digestHeader; + if (digestHeader) + headers[i++] = digestHeader; headers[i] = nullptr; - + ErrCode_ = RequestGet(urlStr.c_str(), headers, PersistentConn_); - + if (ErrCode_) { - Disconnect(); - return false; - } - + Disconnect(); + return false; + } + TString urlBaseStr = URL_.PrintS(THttpURL::FlagNoFrag); - - clearReader(); + + clearReader(); Reader_ = new httpAgentReader(*this, urlBaseStr.c_str(), !PersistentConn_, !Digest_.empty()); - + if (Reader_->readHeader()) { - //mReader->getHeader()->Print(); + //mReader->getHeader()->Print(); if (getHeader()->http_status == HTTP_UNAUTHORIZED && step < 1 && Digest_.processHeader(getAuthHeader(), urlStr.c_str(), "GET")) { - //mReader->skipTheRest(); + //mReader->skipTheRest(); delete Reader_; Reader_ = nullptr; ErrCode_ = 0; - Disconnect(); - continue; - } - - return true; - } - Disconnect(); - clearReader(); - - return false; - } - + Disconnect(); + continue; + } + + return true; + } + Disconnect(); + clearReader(); + + return false; + } + ErrCode_ = HTTP_UNAUTHORIZED; - return false; -} - -/************************************************************/ -/************************************************************/ + return false; +} + +/************************************************************/ +/************************************************************/ diff --git a/library/cpp/http/fetch/httpload.h b/library/cpp/http/fetch/httpload.h index 1441dd27b5..e22e4b809e 100644 --- a/library/cpp/http/fetch/httpload.h +++ b/library/cpp/http/fetch/httpload.h @@ -1,226 +1,226 @@ #pragma once - + #include "httpagent.h" #include "httpparser.h" #include "http_digest.h" -#include <util/system/compat.h> -#include <util/string/vector.h> +#include <util/system/compat.h> +#include <util/string/vector.h> #include <util/network/ip.h> #include <library/cpp/uri/http_url.h> #include <library/cpp/http/misc/httpcodes.h> - -/********************************************************/ -// Section 1: socket handlers -/********************************************************/ -// The following classes allows to adopt template scheme -// THttpAgent for work with socket by flexible -// object-style scheme. - -/********************************************************/ -// This class is used as a base one for flexible -// socket handling + +/********************************************************/ +// Section 1: socket handlers +/********************************************************/ +// The following classes allows to adopt template scheme +// THttpAgent for work with socket by flexible +// object-style scheme. + +/********************************************************/ +// This class is used as a base one for flexible +// socket handling class socketAbstractHandler { -public: - virtual bool Good() = 0; - +public: + virtual bool Good() = 0; + virtual int Connect(const TAddrList& addrs, TDuration Timeout) = 0; - - virtual void Disconnect() = 0; - - virtual void shutdown() = 0; - - virtual bool send(const char* message, ssize_t messlen) = 0; - - virtual bool peek() = 0; - - virtual ssize_t read(void* buffer, ssize_t buflen) = 0; - + + virtual void Disconnect() = 0; + + virtual void shutdown() = 0; + + virtual bool send(const char* message, ssize_t messlen) = 0; + + virtual bool peek() = 0; + + virtual ssize_t read(void* buffer, ssize_t buflen) = 0; + virtual ~socketAbstractHandler() { } - -protected: + +protected: socketAbstractHandler() { } -}; - -/********************************************************/ -// This class is used as a proxy between THttpAgent and -// socketAbstractHandler -// (it is used by template scheme, -// so it does not have virtual methods) +}; + +/********************************************************/ +// This class is used as a proxy between THttpAgent and +// socketAbstractHandler +// (it is used by template scheme, +// so it does not have virtual methods) class TSocketHandlerPtr { -protected: +protected: socketAbstractHandler* Handler_; - -public: + +public: TSocketHandlerPtr() : Handler_(nullptr) { } - + virtual ~TSocketHandlerPtr() { delete Handler_; } - + int Good() { return (Handler_ && Handler_->Good()); } - + int Connect(const TAddrList& addrs, TDuration Timeout) { return (Handler_) ? Handler_->Connect(addrs, Timeout) : 1; - } - + } + void Disconnect() { if (Handler_) Handler_->Disconnect(); - } - + } + void shutdown() { if (Handler_) Handler_->shutdown(); - } - + } + bool send(const char* message, ssize_t messlen) { return (Handler_) ? Handler_->send(message, messlen) : false; - } - + } + virtual bool peek() { return (Handler_) ? Handler_->peek() : false; - } - + } + virtual ssize_t read(void* buffer, ssize_t buflen) { return (Handler_) ? Handler_->read(buffer, buflen) : 0; - } - + } + void setHandler(socketAbstractHandler* handler) { if (Handler_) delete Handler_; Handler_ = handler; - } -}; - -/********************************************************/ -// Here is httpAgent that uses socketAbstractHandler class -// ant its derivatives + } +}; + +/********************************************************/ +// Here is httpAgent that uses socketAbstractHandler class +// ant its derivatives using httpSpecialAgent = THttpAgent<TSocketHandlerPtr>; - -/********************************************************/ -// Regular handler is used as implementation of -// socketAbstractHandler for work through HTTP protocol + +/********************************************************/ +// Regular handler is used as implementation of +// socketAbstractHandler for work through HTTP protocol class socketRegularHandler: public socketAbstractHandler { protected: TSimpleSocketHandler Socket_; - + public: socketRegularHandler() : Socket_() { } - + bool Good() override { return Socket_.Good(); - } - + } + int Connect(const TAddrList& addrs, TDuration Timeout) override { return Socket_.Connect(addrs, Timeout); - } - + } + void Disconnect() override { Socket_.Disconnect(); - } - + } + void shutdown() override { - //Do not block writing to socket - //There are servers that works in a bad way with this - //mSocket.shutdown(); - } - + //Do not block writing to socket + //There are servers that works in a bad way with this + //mSocket.shutdown(); + } + bool send(const char* message, ssize_t messlen) override { return Socket_.send(message, messlen); - } - + } + bool peek() override { return Socket_.peek(); - } - + } + ssize_t read(void* buffer, ssize_t buflen) override { return Socket_.read(buffer, buflen); - } -}; - -/********************************************************/ -// The base factory that allows to choose an appropriate -// socketAbstractHandler implementation by url schema + } +}; + +/********************************************************/ +// The base factory that allows to choose an appropriate +// socketAbstractHandler implementation by url schema class socketHandlerFactory { public: virtual ~socketHandlerFactory() { } - - //returns mHandler_HTTP for correct HTTP-based url + + //returns mHandler_HTTP for correct HTTP-based url virtual socketAbstractHandler* chooseHandler(const THttpURL& url); - - static socketHandlerFactory sInstance; -}; - -/********************************************************/ -// Section 2: the configurates tool to parse an HTTP-response -/********************************************************/ - + + static socketHandlerFactory sInstance; +}; + +/********************************************************/ +// Section 2: the configurates tool to parse an HTTP-response +/********************************************************/ + class httpAgentReader: public THttpParserGeneric<1> { protected: THttpAuthHeader Header_; httpSpecialAgent& Agent_; - + char* Buffer_; void* BufPtr_; int BufSize_; long BufRest_; - - void readBuf(); - + + void readBuf(); + bool step() { if (BufRest_ == 0) - readBuf(); - if (eof()) - return false; - return true; - } - + readBuf(); + if (eof()) + return false; + return true; + } + public: httpAgentReader(httpSpecialAgent& agent, const char* baseUrl, bool assumeConnectionClosed, bool use_auth = false, int bufSize = 0x1000); - - ~httpAgentReader(); - + + ~httpAgentReader(); + bool eof() { return BufRest_ < 0; - } - + } + int error() { return Header_.error; - } - + } + void setError(int errCode) { Header_.error = errCode; - } - + } + const THttpAuthHeader* getAuthHeader() { return &Header_; - } - + } + const THttpHeader* readHeader(); long readPortion(void*& buf); bool skipTheRest(); -}; - -/********************************************************/ -// Section 3: the main class -/********************************************************/ +}; + +/********************************************************/ +// Section 3: the main class +/********************************************************/ class httpLoadAgent: public httpSpecialAgent { protected: socketHandlerFactory& Factory_; @@ -232,76 +232,76 @@ protected: int ErrCode_; char* RealHost_; httpDigestHandler Digest_; - - void clearReader(); + + void clearReader(); bool doSetHost(const TAddrList& addrs); - bool doStartRequest(); - + bool doStartRequest(); + public: httpLoadAgent(bool handleAuthorization = false, socketHandlerFactory& factory = socketHandlerFactory::sInstance); - ~httpLoadAgent(); - - void setRealHost(const char* host); + ~httpLoadAgent(); + + void setRealHost(const char* host); void setIMS(const char* ifModifiedSince); - void addHeaderInstruction(const char* instr); - void dropHeaderInstructions(); - - bool startRequest(const char* url, + void addHeaderInstruction(const char* instr); + void dropHeaderInstructions(); + + bool startRequest(const char* url, const char* url_to_merge = nullptr, bool persistent = false, const TAddrList& addrs = TAddrList()); - + // deprecated v4-only bool startRequest(const char* url, const char* url_to_merge, bool persistent, ui32 ip); - bool startRequest(const THttpURL& url, + bool startRequest(const THttpURL& url, bool persistent = false, const TAddrList& addrs = TAddrList()); - + bool setHost(const char* host_url, const TAddrList& addrs = TAddrList()); - - bool startOneRequest(const char* local_url); - + + bool startOneRequest(const char* local_url); + const THttpAuthHeader* getAuthHeader() { if (Reader_ && Reader_->getAuthHeader()->use_auth) return Reader_->getAuthHeader(); return nullptr; - } - + } + const THttpHeader* getHeader() { if (Reader_) return Reader_->getAuthHeader(); return nullptr; - } - + } + const THttpURL& getURL() { return URL_; - } - + } + bool eof() { if (Reader_) return Reader_->eof(); - return true; - } - + return true; + } + int error() { if (ErrCode_) return ErrCode_; if (Reader_) return Reader_->error(); - return HTTP_BAD_URL; - } - + return HTTP_BAD_URL; + } + long readPortion(void*& buf) { if (Reader_) return Reader_->readPortion(buf); - return -1; - } -}; - -/********************************************************/ + return -1; + } +}; + +/********************************************************/ diff --git a/library/cpp/http/fetch/httpparser.h b/library/cpp/http/fetch/httpparser.h index b666707038..769828e4ae 100644 --- a/library/cpp/http/fetch/httpparser.h +++ b/library/cpp/http/fetch/httpparser.h @@ -1,14 +1,14 @@ #pragma once - -#include "httpfsm.h" -#include "httpheader.h" - + +#include "httpfsm.h" +#include "httpheader.h" + #include <library/cpp/mime/types/mime.h> #include <util/system/yassert.h> #include <library/cpp/http/misc/httpcodes.h> template <size_t headermax = 100 << 10, size_t bodymax = 1 << 20> -struct TFakeCheck { +struct TFakeCheck { bool Check(THttpHeader* /*header*/) { return false; } @@ -22,351 +22,351 @@ struct TFakeCheck { size_t GetMaxBodySize(THttpHeader*) { return bodymax; } -}; - -class THttpParserBase { -public: - enum States { - hp_error, - hp_eof, - hp_in_header, - hp_read_alive, - hp_read_closed, - hp_begin_chunk_header, - hp_chunk_header, - hp_read_chunk - }; - +}; + +class THttpParserBase { +public: + enum States { + hp_error, + hp_eof, + hp_in_header, + hp_read_alive, + hp_read_closed, + hp_begin_chunk_header, + hp_chunk_header, + hp_read_chunk + }; + States GetState() { return State; } - + void setAssumeConnectionClosed(int value) { - AssumeConnectionClosed = value; - } - + AssumeConnectionClosed = value; + } + THttpHeader* GetHttpHeader() const { return Header; } - -protected: - int CheckHeaders() { + +protected: + int CheckHeaders() { if (Header->http_status < HTTP_OK || Header->http_status == HTTP_NO_CONTENT || Header->http_status == HTTP_NOT_MODIFIED) { - Header->content_length = 0; - Header->transfer_chunked = 0; - } - if (Header->transfer_chunked < -1) { - Header->error = HTTP_BAD_ENCODING; - return 1; - } else if (Header->transfer_chunked == -1) { - Header->transfer_chunked = 0; - } - if (!Header->transfer_chunked && Header->content_length < -1) { - Header->error = HTTP_BAD_CONTENT_LENGTH; - return 1; - } + Header->content_length = 0; + Header->transfer_chunked = 0; + } + if (Header->transfer_chunked < -1) { + Header->error = HTTP_BAD_ENCODING; + return 1; + } else if (Header->transfer_chunked == -1) { + Header->transfer_chunked = 0; + } + if (!Header->transfer_chunked && Header->content_length < -1) { + Header->error = HTTP_BAD_CONTENT_LENGTH; + return 1; + } if (Header->http_status == HTTP_OK) { - if (Header->compression_method != HTTP_COMPRESSION_UNSET && - Header->compression_method != HTTP_COMPRESSION_IDENTITY && - Header->compression_method != HTTP_COMPRESSION_GZIP && + if (Header->compression_method != HTTP_COMPRESSION_UNSET && + Header->compression_method != HTTP_COMPRESSION_IDENTITY && + Header->compression_method != HTTP_COMPRESSION_GZIP && Header->compression_method != HTTP_COMPRESSION_DEFLATE) { - Header->error = HTTP_BAD_CONTENT_ENCODING; - return 1; - } - } - if (Header->connection_closed == -1) - Header->connection_closed = (Header->http_minor == 0 || - AssumeConnectionClosed); + Header->error = HTTP_BAD_CONTENT_ENCODING; + return 1; + } + } + if (Header->connection_closed == -1) + Header->connection_closed = (Header->http_minor == 0 || + AssumeConnectionClosed); if (!Header->transfer_chunked && !Header->connection_closed && Header->content_length < 0 && !HeadRequest) { - Header->error = HTTP_LENGTH_UNKNOWN; - return 1; - } - if (Header->http_time < 0) - Header->http_time = 0; - if (Header->mime_type < 0) - Header->mime_type = MIME_UNKNOWN; - return 0; - } - - THttpHeaderParser HeaderParser; - THttpChunkParser ChunkParser; - States State; - long ChunkSize; + Header->error = HTTP_LENGTH_UNKNOWN; + return 1; + } + if (Header->http_time < 0) + Header->http_time = 0; + if (Header->mime_type < 0) + Header->mime_type = MIME_UNKNOWN; + return 0; + } + + THttpHeaderParser HeaderParser; + THttpChunkParser ChunkParser; + States State; + long ChunkSize; THttpHeader* Header; - int AssumeConnectionClosed; - bool HeadRequest; -}; - + int AssumeConnectionClosed; + bool HeadRequest; +}; + template <int isReader, typename TCheck = TFakeCheck<>> -class THttpParserGeneric: public THttpParserBase, public TCheck { -protected: +class THttpParserGeneric: public THttpParserBase, public TCheck { +protected: long ParseGeneric(void*& buf, long& size) { - if (!size) { - switch (State) { - case hp_error: - case hp_eof: - break; - case hp_read_closed: - State = hp_eof; - break; - case hp_in_header: - Header->error = HTTP_HEADER_EOF; - State = hp_error; - break; - case hp_read_alive: - case hp_read_chunk: - if (HeadRequest) - State = hp_eof; - else { - Header->error = HTTP_MESSAGE_EOF; - State = hp_error; - } - break; - case hp_begin_chunk_header: - case hp_chunk_header: - if (HeadRequest) - State = hp_eof; - else { - Header->error = HTTP_CHUNK_EOF; - State = hp_error; - } - break; - } - return 0; - } - while (size) { - int ret; - - switch (State) { - case hp_error: - return 0; - - case hp_eof: - return 0; - - case hp_in_header: - if ((ret = HeaderParser.Execute(buf, size)) < 0) { - Header->error = HTTP_BAD_HEADER_STRING; - State = hp_error; - return 0; - } else if (ret == 2) { - Header->header_size += i32(HeaderParser.lastchar - (char*)buf + 1); - size -= long(HeaderParser.lastchar - (char*)buf + 1); - buf = HeaderParser.lastchar + 1; - State = CheckHeaders() ? hp_error + if (!size) { + switch (State) { + case hp_error: + case hp_eof: + break; + case hp_read_closed: + State = hp_eof; + break; + case hp_in_header: + Header->error = HTTP_HEADER_EOF; + State = hp_error; + break; + case hp_read_alive: + case hp_read_chunk: + if (HeadRequest) + State = hp_eof; + else { + Header->error = HTTP_MESSAGE_EOF; + State = hp_error; + } + break; + case hp_begin_chunk_header: + case hp_chunk_header: + if (HeadRequest) + State = hp_eof; + else { + Header->error = HTTP_CHUNK_EOF; + State = hp_error; + } + break; + } + return 0; + } + while (size) { + int ret; + + switch (State) { + case hp_error: + return 0; + + case hp_eof: + return 0; + + case hp_in_header: + if ((ret = HeaderParser.Execute(buf, size)) < 0) { + Header->error = HTTP_BAD_HEADER_STRING; + State = hp_error; + return 0; + } else if (ret == 2) { + Header->header_size += i32(HeaderParser.lastchar - (char*)buf + 1); + size -= long(HeaderParser.lastchar - (char*)buf + 1); + buf = HeaderParser.lastchar + 1; + State = CheckHeaders() ? hp_error : Header->transfer_chunked ? hp_begin_chunk_header : Header->content_length == 0 ? hp_eof : Header->content_length > 0 ? hp_read_alive : hp_read_closed; - if (State == hp_begin_chunk_header) { - // unget \n for chunk reader + if (State == hp_begin_chunk_header) { + // unget \n for chunk reader buf = (char*)buf - 1; - size++; - } - if (isReader) - return size; - } else { - Header->header_size += size; - size = 0; - } - break; - - case hp_read_alive: - Header->entity_size += size; + size++; + } + if (isReader) + return size; + } else { + Header->header_size += size; + size = 0; + } + break; + + case hp_read_alive: + Header->entity_size += size; if (Header->entity_size >= Header->content_length) { - State = hp_eof; - } + State = hp_eof; + } - TCheck::CheckDocPart(buf, size, Header); - if (isReader) - return size; - size = 0; - break; - - case hp_read_closed: - Header->entity_size += size; - TCheck::CheckDocPart(buf, size, Header); - if (isReader) - return size; - size = 0; - break; - - case hp_begin_chunk_header: - ChunkParser.Init(); - State = hp_chunk_header; + TCheck::CheckDocPart(buf, size, Header); + if (isReader) + return size; + size = 0; + break; + + case hp_read_closed: + Header->entity_size += size; + TCheck::CheckDocPart(buf, size, Header); + if (isReader) + return size; + size = 0; + break; + + case hp_begin_chunk_header: + ChunkParser.Init(); + State = hp_chunk_header; [[fallthrough]]; - - case hp_chunk_header: - if ((ret = ChunkParser.Execute(buf, size)) < 0) { - Header->error = i16(ret == -2 ? HTTP_CHUNK_TOO_LARGE : HTTP_BAD_CHUNK); - State = hp_error; - return 0; - } else if (ret == 2) { - Header->entity_size += i32(ChunkParser.lastchar - (char*)buf + 1); - size -= long(ChunkParser.lastchar - (char*)buf + 1); - buf = ChunkParser.lastchar + 1; - ChunkSize = ChunkParser.chunk_length; + + case hp_chunk_header: + if ((ret = ChunkParser.Execute(buf, size)) < 0) { + Header->error = i16(ret == -2 ? HTTP_CHUNK_TOO_LARGE : HTTP_BAD_CHUNK); + State = hp_error; + return 0; + } else if (ret == 2) { + Header->entity_size += i32(ChunkParser.lastchar - (char*)buf + 1); + size -= long(ChunkParser.lastchar - (char*)buf + 1); + buf = ChunkParser.lastchar + 1; + ChunkSize = ChunkParser.chunk_length; Y_ASSERT(ChunkSize >= 0); - State = ChunkSize ? hp_read_chunk : hp_eof; - } else { - Header->entity_size += size; - size = 0; - } - break; - - case hp_read_chunk: - if (size >= ChunkSize) { - Header->entity_size += ChunkSize; - State = hp_begin_chunk_header; - TCheck::CheckDocPart(buf, ChunkSize, Header); - if (isReader) - return ChunkSize; - size -= ChunkSize; + State = ChunkSize ? hp_read_chunk : hp_eof; + } else { + Header->entity_size += size; + size = 0; + } + break; + + case hp_read_chunk: + if (size >= ChunkSize) { + Header->entity_size += ChunkSize; + State = hp_begin_chunk_header; + TCheck::CheckDocPart(buf, ChunkSize, Header); + if (isReader) + return ChunkSize; + size -= ChunkSize; buf = (char*)buf + ChunkSize; - } else { - Header->entity_size += size; - ChunkSize -= size; - TCheck::CheckDocPart(buf, size, Header); - if (isReader) - return size; - size = 0; - } + } else { + Header->entity_size += size; + ChunkSize -= size; + TCheck::CheckDocPart(buf, size, Header); + if (isReader) + return size; + size = 0; + } break; - } - } - return size; - } -}; - + } + } + return size; + } +}; + template <class TCheck = TFakeCheck<>> -class THttpParser: public THttpParserGeneric<0, TCheck> { - typedef THttpParserGeneric<0, TCheck> TBaseT; //sorry avoiding gcc 3.4.6 BUG! -public: +class THttpParser: public THttpParserGeneric<0, TCheck> { + typedef THttpParserGeneric<0, TCheck> TBaseT; //sorry avoiding gcc 3.4.6 BUG! +public: void Init(THttpHeader* H, bool head_request = false) { - TBaseT::Header = H; - TBaseT::HeaderParser.Init(TBaseT::Header); - TBaseT::State = TBaseT::hp_in_header; - TBaseT::AssumeConnectionClosed = 0; - TBaseT::HeadRequest = head_request; - } - + TBaseT::Header = H; + TBaseT::HeaderParser.Init(TBaseT::Header); + TBaseT::State = TBaseT::hp_in_header; + TBaseT::AssumeConnectionClosed = 0; + TBaseT::HeadRequest = head_request; + } + void Parse(void* buf, long size) { TBaseT::ParseGeneric(buf, size); - } -}; - -class TMemoReader { -public: + } +}; + +class TMemoReader { +public: int Init(void* buf, long bufsize) { - Buf = buf; - Bufsize = bufsize; - return 0; - } + Buf = buf; + Bufsize = bufsize; + return 0; + } long Read(void*& buf) { Y_ASSERT(Bufsize >= 0); - if (!Bufsize) { - Bufsize = -1; - return 0; - } - buf = Buf; - long ret = Bufsize; - Bufsize = 0; - return ret; - } - -protected: - long Bufsize; + if (!Bufsize) { + Bufsize = -1; + return 0; + } + buf = Buf; + long ret = Bufsize; + Bufsize = 0; + return ret; + } + +protected: + long Bufsize; void* Buf; -}; - -template <class Reader> -class THttpReader: public THttpParserGeneric<1>, public Reader { - typedef THttpParserGeneric<1> TBaseT; +}; + +template <class Reader> +class THttpReader: public THttpParserGeneric<1>, public Reader { + typedef THttpParserGeneric<1> TBaseT; -public: +public: using TBaseT::AssumeConnectionClosed; - using TBaseT::Header; - using TBaseT::ParseGeneric; - using TBaseT::State; - + using TBaseT::Header; + using TBaseT::ParseGeneric; + using TBaseT::State; + int Init(THttpHeader* H, int parsHeader, int assumeConnectionClosed = 0, bool headRequest = false) { - Header = H; - Eoferr = 1; - Size = 0; + Header = H; + Eoferr = 1; + Size = 0; AssumeConnectionClosed = assumeConnectionClosed; HeadRequest = headRequest; - return parsHeader ? ParseHeader() : SkipHeader(); - } - + return parsHeader ? ParseHeader() : SkipHeader(); + } + long Read(void*& buf) { - long Chunk; - do { - if (!Size) { - if (Eoferr != 1) - return Eoferr; - else if ((Size = (long)Reader::Read(Ptr)) < 0) { - Header->error = HTTP_CONNECTION_LOST; - return Eoferr = -1; - } - } + long Chunk; + do { + if (!Size) { + if (Eoferr != 1) + return Eoferr; + else if ((Size = (long)Reader::Read(Ptr)) < 0) { + Header->error = HTTP_CONNECTION_LOST; + return Eoferr = -1; + } + } Chunk = ParseGeneric(Ptr, Size); - buf = Ptr; - Ptr = (char*)Ptr + Chunk; - Size -= Chunk; + buf = Ptr; + Ptr = (char*)Ptr + Chunk; + Size -= Chunk; if (State == hp_eof) { Size = 0; - Eoferr = 0; + Eoferr = 0; } else if (State == hp_error) - return Eoferr = -1; - } while (!Chunk); - return Chunk; - } - -protected: - int ParseHeader() { - HeaderParser.Init(Header); - State = hp_in_header; - while (State == hp_in_header) { - if ((Size = (long)Reader::Read(Ptr)) < 0) - return Eoferr = -1; + return Eoferr = -1; + } while (!Chunk); + return Chunk; + } + +protected: + int ParseHeader() { + HeaderParser.Init(Header); + State = hp_in_header; + while (State == hp_in_header) { + if ((Size = (long)Reader::Read(Ptr)) < 0) + return Eoferr = -1; ParseGeneric(Ptr, Size); - } - if (State == hp_error) - return Eoferr = -1; - if (State == hp_eof) - Eoferr = 0; - return 0; - } - - int SkipHeader() { - long hdrsize = Header->header_size; - while (hdrsize) { - if ((Size = (long)Reader::Read(Ptr)) <= 0) - return Eoferr = -1; - if (Size >= hdrsize) { - Size -= hdrsize; - Ptr = (char*)Ptr + hdrsize; - break; - } - hdrsize -= Size; - } - State = Header->transfer_chunked ? hp_begin_chunk_header + } + if (State == hp_error) + return Eoferr = -1; + if (State == hp_eof) + Eoferr = 0; + return 0; + } + + int SkipHeader() { + long hdrsize = Header->header_size; + while (hdrsize) { + if ((Size = (long)Reader::Read(Ptr)) <= 0) + return Eoferr = -1; + if (Size >= hdrsize) { + Size -= hdrsize; + Ptr = (char*)Ptr + hdrsize; + break; + } + hdrsize -= Size; + } + State = Header->transfer_chunked ? hp_begin_chunk_header : Header->content_length == 0 ? hp_eof : Header->content_length > 0 ? hp_read_alive : hp_read_closed; - Header->entity_size = 0; - if (State == hp_eof) - Eoferr = 0; - else if (State == hp_begin_chunk_header) { - // unget \n for chunk reader - Ptr = (char*)Ptr - 1; - ++Size; - } - return 0; - } - + Header->entity_size = 0; + if (State == hp_eof) + Eoferr = 0; + else if (State == hp_begin_chunk_header) { + // unget \n for chunk reader + Ptr = (char*)Ptr - 1; + ++Size; + } + return 0; + } + void* Ptr; - long Size; + long Size; int Eoferr; -}; +}; diff --git a/library/cpp/http/fetch/httpzreader.h b/library/cpp/http/fetch/httpzreader.h index d951d21e9a..68eb00853d 100644 --- a/library/cpp/http/fetch/httpzreader.h +++ b/library/cpp/http/fetch/httpzreader.h @@ -1,55 +1,55 @@ #pragma once - + #include "httpheader.h" #include "httpparser.h" #include "exthttpcodes.h" - + #include <util/system/defaults.h> #include <util/generic/yexception.h> -#include <contrib/libs/zlib/zlib.h> - +#include <contrib/libs/zlib/zlib.h> + #include <errno.h> -#ifndef ENOTSUP +#ifndef ENOTSUP #define ENOTSUP 45 -#endif - -template <class Reader> -class TCompressedHttpReader: public THttpReader<Reader> { - typedef THttpReader<Reader> TBase; +#endif + +template <class Reader> +class TCompressedHttpReader: public THttpReader<Reader> { + typedef THttpReader<Reader> TBase; -public: +public: using TBase::AssumeConnectionClosed; - using TBase::Header; - using TBase::ParseGeneric; - using TBase::State; - + using TBase::Header; + using TBase::ParseGeneric; + using TBase::State; + static constexpr size_t DefaultBufSize = 64 << 10; static constexpr unsigned int DefaultWinSize = 15; - TCompressedHttpReader() - : CompressedInput(false) - , BufSize(0) - , CurContSize(0) - , MaxContSize(0) + TCompressedHttpReader() + : CompressedInput(false) + , BufSize(0) + , CurContSize(0) + , MaxContSize(0) , Buf(nullptr) - , ZErr(0) + , ZErr(0) , ConnectionClosed(0) , IgnoreTrailingGarbage(true) - { - memset(&Stream, 0, sizeof(Stream)); - } - - ~TCompressedHttpReader() { - ClearStream(); - - if (Buf) { - free(Buf); + { + memset(&Stream, 0, sizeof(Stream)); + } + + ~TCompressedHttpReader() { + ClearStream(); + + if (Buf) { + free(Buf); Buf = nullptr; - } - } - + } + } + void SetConnectionClosed(int cc) { ConnectionClosed = cc; } @@ -66,196 +66,196 @@ public: const unsigned int winSize = DefaultWinSize, bool headRequest = false) { - ZErr = 0; - CurContSize = 0; - MaxContSize = maxContSize; - + ZErr = 0; + CurContSize = 0; + MaxContSize = maxContSize; + int ret = TBase::Init(H, parsHeader, ConnectionClosed, headRequest); - if (ret) - return ret; - - ret = SetCompression(H->compression_method, bufSize, winSize); - return ret; - } - + if (ret) + return ret; + + ret = SetCompression(H->compression_method, bufSize, winSize); + return ret; + } + long Read(void*& buf) { - if (!CompressedInput) { - long res = TBase::Read(buf); - if (res > 0) { - CurContSize += (size_t)res; - if (CurContSize > MaxContSize) { - ZErr = E2BIG; - return -1; - } - } - return res; - } - - while (true) { - if (Stream.avail_in == 0) { + if (!CompressedInput) { + long res = TBase::Read(buf); + if (res > 0) { + CurContSize += (size_t)res; + if (CurContSize > MaxContSize) { + ZErr = E2BIG; + return -1; + } + } + return res; + } + + while (true) { + if (Stream.avail_in == 0) { void* tmpin = Stream.next_in; long res = TBase::Read(tmpin); Stream.next_in = (Bytef*)tmpin; if (res <= 0) return res; Stream.avail_in = (uInt)res; - } - - Stream.next_out = Buf; - Stream.avail_out = (uInt)BufSize; - buf = Buf; - - int err = inflate(&Stream, Z_SYNC_FLUSH); - + } + + Stream.next_out = Buf; + Stream.avail_out = (uInt)BufSize; + buf = Buf; + + int err = inflate(&Stream, Z_SYNC_FLUSH); + //Y_ASSERT(Stream.avail_in == 0); - - switch (err) { - case Z_OK: - // there is no data in next_out yet - if (BufSize == Stream.avail_out) - continue; + + switch (err) { + case Z_OK: + // there is no data in next_out yet + if (BufSize == Stream.avail_out) + continue; [[fallthrough]]; // don't break or return; continue with Z_STREAM_END case - - case Z_STREAM_END: - if (Stream.total_out > MaxContSize) { - ZErr = E2BIG; - return -1; - } + + case Z_STREAM_END: + if (Stream.total_out > MaxContSize) { + ZErr = E2BIG; + return -1; + } if (!IgnoreTrailingGarbage && BufSize == Stream.avail_out && Stream.avail_in > 0) { Header->error = EXT_HTTP_GZIPERROR; ZErr = EFAULT; Stream.msg = (char*)"trailing garbage"; return -1; } - return long(BufSize - Stream.avail_out); - - case Z_NEED_DICT: - case Z_DATA_ERROR: - Header->error = EXT_HTTP_GZIPERROR; - ZErr = EFAULT; - return -1; - - case Z_MEM_ERROR: - ZErr = ENOMEM; - return -1; - - default: - ZErr = EINVAL; - return -1; - } - } - - return -1; - } - + return long(BufSize - Stream.avail_out); + + case Z_NEED_DICT: + case Z_DATA_ERROR: + Header->error = EXT_HTTP_GZIPERROR; + ZErr = EFAULT; + return -1; + + case Z_MEM_ERROR: + ZErr = ENOMEM; + return -1; + + default: + ZErr = EINVAL; + return -1; + } + } + + return -1; + } + const char* ZMsg() const { - return Stream.msg; - } - - int ZError() const { - return ZErr; - } - - size_t GetCurContSize() const { - return CompressedInput ? Stream.total_out : CurContSize; - } - -protected: - int SetCompression(const int compression, const size_t bufSize, - const unsigned int winSize) { - ClearStream(); - - int winsize = winSize; - switch ((enum HTTP_COMPRESSION)compression) { - case HTTP_COMPRESSION_UNSET: - case HTTP_COMPRESSION_IDENTITY: - CompressedInput = false; - return 0; - case HTTP_COMPRESSION_GZIP: - CompressedInput = true; + return Stream.msg; + } + + int ZError() const { + return ZErr; + } + + size_t GetCurContSize() const { + return CompressedInput ? Stream.total_out : CurContSize; + } + +protected: + int SetCompression(const int compression, const size_t bufSize, + const unsigned int winSize) { + ClearStream(); + + int winsize = winSize; + switch ((enum HTTP_COMPRESSION)compression) { + case HTTP_COMPRESSION_UNSET: + case HTTP_COMPRESSION_IDENTITY: + CompressedInput = false; + return 0; + case HTTP_COMPRESSION_GZIP: + CompressedInput = true; winsize += 16; // 16 indicates gzip, see zlib.h - break; - case HTTP_COMPRESSION_DEFLATE: - CompressedInput = true; - winsize = -winsize; // negative indicates raw deflate stream, see zlib.h - break; - case HTTP_COMPRESSION_COMPRESS: - case HTTP_COMPRESSION_ERROR: - default: - CompressedInput = false; - ZErr = ENOTSUP; - return -1; - } - - if (bufSize != BufSize) { - if (Buf) - free(Buf); - Buf = (ui8*)malloc(bufSize); - if (!Buf) { - ZErr = ENOMEM; - return -1; - } - BufSize = bufSize; - } - - int err = inflateInit2(&Stream, winsize); - switch (err) { - case Z_OK: - Stream.total_in = 0; - Stream.total_out = 0; - Stream.avail_in = 0; - return 0; - - case Z_DATA_ERROR: // never happens, see zlib.h - CompressedInput = false; - ZErr = EFAULT; - return -1; - - case Z_MEM_ERROR: - CompressedInput = false; - ZErr = ENOMEM; - return -1; - - default: - CompressedInput = false; - ZErr = EINVAL; - return -1; - } - } - - void ClearStream() { - if (CompressedInput) { - inflateEnd(&Stream); - CompressedInput = false; - } - } - - z_stream Stream; - bool CompressedInput; - size_t BufSize; - size_t CurContSize, MaxContSize; - ui8* Buf; - int ZErr; + break; + case HTTP_COMPRESSION_DEFLATE: + CompressedInput = true; + winsize = -winsize; // negative indicates raw deflate stream, see zlib.h + break; + case HTTP_COMPRESSION_COMPRESS: + case HTTP_COMPRESSION_ERROR: + default: + CompressedInput = false; + ZErr = ENOTSUP; + return -1; + } + + if (bufSize != BufSize) { + if (Buf) + free(Buf); + Buf = (ui8*)malloc(bufSize); + if (!Buf) { + ZErr = ENOMEM; + return -1; + } + BufSize = bufSize; + } + + int err = inflateInit2(&Stream, winsize); + switch (err) { + case Z_OK: + Stream.total_in = 0; + Stream.total_out = 0; + Stream.avail_in = 0; + return 0; + + case Z_DATA_ERROR: // never happens, see zlib.h + CompressedInput = false; + ZErr = EFAULT; + return -1; + + case Z_MEM_ERROR: + CompressedInput = false; + ZErr = ENOMEM; + return -1; + + default: + CompressedInput = false; + ZErr = EINVAL; + return -1; + } + } + + void ClearStream() { + if (CompressedInput) { + inflateEnd(&Stream); + CompressedInput = false; + } + } + + z_stream Stream; + bool CompressedInput; + size_t BufSize; + size_t CurContSize, MaxContSize; + ui8* Buf; + int ZErr; int ConnectionClosed; bool IgnoreTrailingGarbage; -}; - +}; + class zlib_exception: public yexception { -}; - -template <class Reader> -class SCompressedHttpReader: public TCompressedHttpReader<Reader> { - typedef TCompressedHttpReader<Reader> TBase; - -public: - using TBase::ZError; - using TBase::ZMsg; - - SCompressedHttpReader() +}; + +template <class Reader> +class SCompressedHttpReader: public TCompressedHttpReader<Reader> { + typedef TCompressedHttpReader<Reader> TBase; + +public: + using TBase::ZError; + using TBase::ZMsg; + + SCompressedHttpReader() : TBase() { } - + int Init( THttpHeader* H, int parsHeader, @@ -265,31 +265,31 @@ public: bool headRequest = false) { int ret = TBase::Init(H, parsHeader, maxContSize, bufSize, winSize, headRequest); - return (int)HandleRetValue((long)ret); - } - + return (int)HandleRetValue((long)ret); + } + long Read(void*& buf) { - long ret = TBase::Read(buf); - return HandleRetValue(ret); - } - -protected: - long HandleRetValue(long ret) { - switch (ZError()) { - case 0: - return ret; - case ENOMEM: + long ret = TBase::Read(buf); + return HandleRetValue(ret); + } + +protected: + long HandleRetValue(long ret) { + switch (ZError()) { + case 0: + return ret; + case ENOMEM: ythrow yexception() << "SCompressedHttpReader: not enough memory"; - case EINVAL: + case EINVAL: ythrow yexception() << "SCompressedHttpReader: zlib error: " << ZMsg(); - case ENOTSUP: + case ENOTSUP: ythrow yexception() << "SCompressedHttpReader: unsupported compression method"; - case EFAULT: + case EFAULT: ythrow zlib_exception() << "SCompressedHttpReader: " << ZMsg(); - case E2BIG: + case E2BIG: ythrow zlib_exception() << "SCompressedHttpReader: Content exceeds maximum length"; - default: + default: ythrow yexception() << "SCompressedHttpReader: unknown error"; - } - } -}; + } + } +}; diff --git a/library/cpp/http/fetch/ya.make b/library/cpp/http/fetch/ya.make index aa8b073a8c..7737127463 100644 --- a/library/cpp/http/fetch/ya.make +++ b/library/cpp/http/fetch/ya.make @@ -1,5 +1,5 @@ LIBRARY() - + OWNER( g:zora ) @@ -14,25 +14,25 @@ PEERDIR( library/cpp/uri ) -SRCS( - http_digest.cpp - http_socket.cpp +SRCS( + http_digest.cpp + http_socket.cpp httpheader.cpp - httpload.cpp - exthttpcodes.cpp + httpload.cpp + exthttpcodes.cpp httpfsm.rl6 - httpagent.h - httpfetcher.h - httpheader.h - httpparser.h - httpzreader.h + httpagent.h + httpfetcher.h + httpheader.h + httpparser.h + httpzreader.h sockhandler.h -) - +) + GENERATE_ENUM_SERIALIZATION(httpheader.h) SET(RAGEL6_FLAGS -CF1) -END() +END() RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/http/io/chunk_ut.cpp b/library/cpp/http/io/chunk_ut.cpp index 5a365e23f7..da283f8568 100644 --- a/library/cpp/http/io/chunk_ut.cpp +++ b/library/cpp/http/io/chunk_ut.cpp @@ -2,7 +2,7 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/system/tempfile.h> #include <util/stream/null.h> diff --git a/library/cpp/http/io/compression_ut.cpp b/library/cpp/http/io/compression_ut.cpp index 1201cda877..2f3d131f8c 100644 --- a/library/cpp/http/io/compression_ut.cpp +++ b/library/cpp/http/io/compression_ut.cpp @@ -24,7 +24,7 @@ Y_UNIT_TEST_SUITE(THttpCompressionTest) { auto encodedStream = (*encoder)(&buffer); encodedStream->Write(DATA); } - + TZLibDecompress decompressor(&buffer); UNIT_ASSERT_EQUAL(decompressor.ReadAll(), DATA); } diff --git a/library/cpp/http/io/headers.cpp b/library/cpp/http/io/headers.cpp index 070bfbe127..4ec27a29e8 100644 --- a/library/cpp/http/io/headers.cpp +++ b/library/cpp/http/io/headers.cpp @@ -81,7 +81,7 @@ void THttpHeaders::RemoveHeader(const TStringBuf header) { } } -void THttpHeaders::AddOrReplaceHeader(const THttpInputHeader& header) { +void THttpHeaders::AddOrReplaceHeader(const THttpInputHeader& header) { for (auto& hdr : Headers_) { if (AsciiCompareIgnoreCase(hdr.Name(), header.Name()) == 0) { hdr = header; diff --git a/library/cpp/http/io/headers.h b/library/cpp/http/io/headers.h index 9bfc0d0172..a71793d1c6 100644 --- a/library/cpp/http/io/headers.h +++ b/library/cpp/http/io/headers.h @@ -12,7 +12,7 @@ class IOutputStream; /// @addtogroup Streams_HTTP /// @{ /// Объект, содержащий информацию о HTTP-заголовке. -class THttpInputHeader { +class THttpInputHeader { public: /// @param[in] header - строка вида 'параметр: значение'. THttpInputHeader(TStringBuf header); diff --git a/library/cpp/http/io/stream.cpp b/library/cpp/http/io/stream.cpp index 28aa8af048..6689be684f 100644 --- a/library/cpp/http/io/stream.cpp +++ b/library/cpp/http/io/stream.cpp @@ -971,14 +971,14 @@ unsigned ParseHttpRetCode(const TStringBuf& ret) { return FromString<unsigned>(code.data(), code.size()); } - + void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent, const TStringBuf& from) { - TSocketOutput so(s); - THttpOutput output(&so); - - output.EnableKeepAlive(false); - output.EnableCompression(false); - + TSocketOutput so(s); + THttpOutput output(&so); + + output.EnableKeepAlive(false); + output.EnableCompression(false); + const IOutputStream::TPart parts[] = { IOutputStream::TPart(TStringBuf("GET ")), IOutputStream::TPart(request), @@ -994,11 +994,11 @@ void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf IOutputStream::TPart(from), IOutputStream::TPart::CrLf(), IOutputStream::TPart::CrLf(), - }; + }; - output.Write(parts, sizeof(parts) / sizeof(*parts)); - output.Finish(); -} + output.Write(parts, sizeof(parts) / sizeof(*parts)); + output.Finish(); +} TArrayRef<const TStringBuf> SupportedCodings() { return TCompressionCodecFactory::Instance().GetBestCodecs(); diff --git a/library/cpp/http/io/stream.h b/library/cpp/http/io/stream.h index 7a2693427e..78ca4fc814 100644 --- a/library/cpp/http/io/stream.h +++ b/library/cpp/http/io/stream.h @@ -172,7 +172,7 @@ unsigned ParseHttpRetCode(const TStringBuf& ret); /// Отправляет HTTP-серверу запрос с минимумом необходимых заголовков. void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent = "YandexSomething/1.0", const TStringBuf& from = "webadmin@yandex.ru"); - + TArrayRef<const TStringBuf> SupportedCodings(); /// @} diff --git a/library/cpp/http/io/stream_ut.cpp b/library/cpp/http/io/stream_ut.cpp index 5fe494335a..1ea35df675 100644 --- a/library/cpp/http/io/stream_ut.cpp +++ b/library/cpp/http/io/stream_ut.cpp @@ -116,7 +116,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { output.Write(r.data(), r.size()); output.Finish(); } - + { TSocketInput si(s); THttpInput input(&si); @@ -253,7 +253,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { SendMinimalHttpRequest(s, "www.yandex.lo", "/"); - TSocketInput si(s); + TSocketInput si(s); THttpInput input(&si); unsigned httpCode = ParseHttpRetCode(input.FirstLine()); UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); @@ -261,7 +261,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TransferData(&input, &dbg); server.Stop(); } - + Y_UNIT_TEST(TestResponseWithBlanks) { TString res = "qqqqqq\r\n\r\nsdasdsad\r\n"; TPortManager pm; @@ -292,13 +292,13 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TStringOutput strOut(str); TBufferedOutput bufOut(&strOut, 8192); THttpOutput httpOut(&bufOut); - + httpOut.EnableKeepAlive(true); httpOut.EnableCompression(true); const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; httpOut << header; - + unsigned curLen = str.size(); const char* body = "<html>Hello</html>"; httpOut << body; @@ -306,7 +306,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { httpOut.Flush(); UNIT_ASSERT_VALUES_EQUAL(curLen + strlen(body), str.size()); } - + Y_UNIT_TEST(TestOutputPostFlush) { TString str; TString checkStr; diff --git a/library/cpp/http/misc/httpdate.cpp b/library/cpp/http/misc/httpdate.cpp index 719f100451..4a3031bbf4 100644 --- a/library/cpp/http/misc/httpdate.cpp +++ b/library/cpp/http/misc/httpdate.cpp @@ -26,14 +26,14 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include <util/system/defaults.h> +#include <util/system/defaults.h> #include <sys/types.h> -#include <cctype> -#include <cstdio> -#include <cstdlib> -#include <cstring> -#include <ctime> +#include <cctype> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <ctime> #include <util/system/compat.h> /* stricmp */ #include <util/system/yassert.h> diff --git a/library/cpp/http/misc/httpdate.h b/library/cpp/http/misc/httpdate.h index e9583b4230..04876f38fe 100644 --- a/library/cpp/http/misc/httpdate.h +++ b/library/cpp/http/misc/httpdate.h @@ -3,7 +3,7 @@ #include <util/datetime/base.h> #include <util/generic/string.h> -#include <ctime> +#include <ctime> #define BAD_DATE ((time_t)-1) diff --git a/library/cpp/http/misc/httpreqdata.h b/library/cpp/http/misc/httpreqdata.h index ae2652ac42..16e59c4d78 100644 --- a/library/cpp/http/misc/httpreqdata.h +++ b/library/cpp/http/misc/httpreqdata.h @@ -1,5 +1,5 @@ #pragma once - + #include <library/cpp/digest/lower_case/hash_ops.h> #include <util/str_stl.h> @@ -14,11 +14,11 @@ #include <util/generic/string.h> #include <util/datetime/base.h> #include <util/generic/buffer.h> - + using THttpHeadersContainer = THashMap<TString, TString, TCIOps, TCIOps>; class TBaseServerRequestData { -public: +public: TBaseServerRequestData(SOCKET s = INVALID_SOCKET); TBaseServerRequestData(const char* qs, SOCKET s = INVALID_SOCKET); @@ -29,7 +29,7 @@ public: const TString& ServerName() const { return Host; - } + } NAddr::IRemoteAddrPtr ServerAddress() const { return NAddr::GetSockAddr(Socket); @@ -37,15 +37,15 @@ public: const TString& ServerPort() const { return Port; - } + } - const char* ScriptName() const { - return Path; - } + const char* ScriptName() const { + return Path; + } - const char* QueryString() const { - return Search; - } + const char* QueryString() const { + return Search; + } TStringBuf QueryStringBuf() const { return TStringBuf(Search, SearchLength); @@ -82,22 +82,22 @@ public: } void SetPath(const TString& path); - const char* GetCurPage() const; + const char* GetCurPage() const; bool Parse(const char* req); void AddHeader(const TString& name, const TString& value); -private: +private: TBuffer PathStorage; mutable char* Addr; TString Host; TString Port; - char* Path; - char* Search; + char* Path; + char* Search; size_t SearchLength; // length of Search TStringBuf OrigSearch; THttpHeadersContainer HeadersIn_; mutable char AddrData[INET6_ADDRSTRLEN]; - SOCKET Socket; + SOCKET Socket; ui64 BeginTime; mutable TString CurPage; TBuffer ParseBuf; @@ -105,7 +105,7 @@ private: }; class TServerRequestData: public TBaseServerRequestData { -public: +public: TServerRequestData(SOCKET s = INVALID_SOCKET) : TBaseServerRequestData(s) { @@ -122,4 +122,4 @@ public: public: TCgiParameters CgiParam; -}; +}; diff --git a/library/cpp/http/server/http.cpp b/library/cpp/http/server/http.cpp index 8d1900ef9e..128583bdd7 100644 --- a/library/cpp/http/server/http.cpp +++ b/library/cpp/http/server/http.cpp @@ -22,7 +22,7 @@ #include <cerrno> #include <cstring> #include <ctime> - + #include <sys/stat.h> #include <sys/types.h> @@ -139,7 +139,7 @@ public: try { ((TImpl*)param)->ListenSocket(); } catch (...) { - + } return nullptr; @@ -149,7 +149,7 @@ public: THolder<TClientRequest> obj(Cb_->CreateClient()); obj->Conn_.Reset(c.Release()); - + return obj; } @@ -171,7 +171,7 @@ public: void SaveErrorCode() { ErrorCode = WSAGetLastError(); } - + int GetErrorCode() const { return ErrorCode; } @@ -218,7 +218,7 @@ public: ListenThread.Reset(nullptr); } } - + void Stop() { Shutdown(); @@ -309,7 +309,7 @@ public: Server_->AddRequestFromSocket(s, TInstant::Now(), SockAddrRef_); } - + SOCKET GetSocket() const noexcept { return S_; } @@ -338,13 +338,13 @@ public: return; } - + Requests->Start(Options_.nThreads, Options_.MaxQueueSize); FailRequests->Start(Options_.nFThreads, Options_.MaxFQueueSize); Cb_->OnListenStart(); ListenerRunningOK = true; ListenStartEvent.Signal(); - + TVector<void*> events; events.resize(1); @@ -378,7 +378,7 @@ public: Cb_->OnException(); } } - + while (!Reqs.Empty()) { THolder<TListenSocket> ls(Reqs.PopFront()); @@ -586,20 +586,20 @@ void TClientConnection::OnPollEvent(TInstant now) { return; } - } + } THolder<TClientRequest> obj(HttpServ_->CreateRequest(this_)); AcceptMoment = now; HttpServ_->AddRequest(obj, Reject_); -} - +} + void TClientConnection::Activate(TInstant now) noexcept { HttpServ_->Connections->Erase(this, now); LastUsed = now; ++ReceivedRequests; -} - +} + void TClientConnection::DeActivate() { HttpServ_->Connections->Add(this); } diff --git a/library/cpp/http/server/http.h b/library/cpp/http/server/http.h index 7fb210397c..b292d38f27 100644 --- a/library/cpp/http/server/http.h +++ b/library/cpp/http/server/http.h @@ -1,5 +1,5 @@ #pragma once - + #include "conn.h" #include "options.h" @@ -97,8 +97,8 @@ private: private: THolder<TImpl> Impl_; -}; - +}; + /** * @deprecated Use TRequestReplier instead */ diff --git a/library/cpp/ipv6_address/ipv6_address.h b/library/cpp/ipv6_address/ipv6_address.h index 458ba87122..1d7eb0b65f 100644 --- a/library/cpp/ipv6_address/ipv6_address.h +++ b/library/cpp/ipv6_address/ipv6_address.h @@ -19,8 +19,8 @@ namespace NAddr { class IRemoteAddr; } -struct in6_addr; -struct in_addr; +struct in6_addr; +struct in_addr; struct sockaddr; struct sockaddr_in; struct sockaddr_in6; diff --git a/library/cpp/lfalloc/ya.make b/library/cpp/lfalloc/ya.make index 05dad5c2af..cace05f9d8 100644 --- a/library/cpp/lfalloc/ya.make +++ b/library/cpp/lfalloc/ya.make @@ -20,6 +20,6 @@ PEERDIR( library/cpp/malloc/api ) -SET(IDE_FOLDER "util") +SET(IDE_FOLDER "util") END() diff --git a/library/cpp/lfalloc/yt/ya.make b/library/cpp/lfalloc/yt/ya.make index fa5791fcf9..8c1a4f8a72 100644 --- a/library/cpp/lfalloc/yt/ya.make +++ b/library/cpp/lfalloc/yt/ya.make @@ -24,6 +24,6 @@ PEERDIR( library/cpp/malloc/api ) -SET(IDE_FOLDER "util") +SET(IDE_FOLDER "util") END() diff --git a/library/cpp/logger/backend.h b/library/cpp/logger/backend.h index 0b64a80be5..d088726d6d 100644 --- a/library/cpp/logger/backend.h +++ b/library/cpp/logger/backend.h @@ -1,5 +1,5 @@ #pragma once - + #include "priority.h" #include <util/generic/noncopyable.h> @@ -7,7 +7,7 @@ #include <cstddef> struct TLogRecord; - + // NOTE: be aware that all `TLogBackend`s are registred in singleton. class TLogBackend: public TNonCopyable { public: diff --git a/library/cpp/logger/file.h b/library/cpp/logger/file.h index 9204d5ad60..10b4cd0c20 100644 --- a/library/cpp/logger/file.h +++ b/library/cpp/logger/file.h @@ -1,5 +1,5 @@ #pragma once - + #include "backend.h" #include <util/generic/fwd.h> @@ -16,4 +16,4 @@ public: private: class TImpl; TAtomicSharedPtr<TImpl> Impl_; -}; +}; diff --git a/library/cpp/logger/log.h b/library/cpp/logger/log.h index 35896d6b1e..8be984ccc8 100644 --- a/library/cpp/logger/log.h +++ b/library/cpp/logger/log.h @@ -1,5 +1,5 @@ #pragma once - + #include "backend.h" #include "element.h" #include "priority.h" diff --git a/library/cpp/logger/log_ut.cpp b/library/cpp/logger/log_ut.cpp index a30ae0f0df..8de46f17f5 100644 --- a/library/cpp/logger/log_ut.cpp +++ b/library/cpp/logger/log_ut.cpp @@ -54,7 +54,7 @@ void TLogTest::TestFile() { log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); } - TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); + TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); } @@ -73,7 +73,7 @@ void TLogTest::TestThreaded() { log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); } - TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); + TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); } diff --git a/library/cpp/logger/null.h b/library/cpp/logger/null.h index 3ec8c670d9..a02f250b00 100644 --- a/library/cpp/logger/null.h +++ b/library/cpp/logger/null.h @@ -1,5 +1,5 @@ #pragma once - + #include "backend.h" class TNullLogBackend: public TLogBackend { diff --git a/library/cpp/logger/priority.h b/library/cpp/logger/priority.h index aafa74be8b..d2a9fa0a07 100644 --- a/library/cpp/logger/priority.h +++ b/library/cpp/logger/priority.h @@ -1,5 +1,5 @@ #pragma once - + enum ELogPriority { TLOG_EMERG = 0 /* "EMERG" */, TLOG_ALERT = 1 /* "ALERT" */, diff --git a/library/cpp/logger/record.h b/library/cpp/logger/record.h index dd9d0d03a1..c28a7785fd 100644 --- a/library/cpp/logger/record.h +++ b/library/cpp/logger/record.h @@ -1,5 +1,5 @@ #pragma once - + #include "priority.h" #include <util/system/defaults.h> @@ -8,7 +8,7 @@ struct TLogRecord { const char* Data; size_t Len; ELogPriority Priority; - + inline TLogRecord(ELogPriority priority, const char* data, size_t len) noexcept : Data(data) , Len(len) diff --git a/library/cpp/logger/stream.h b/library/cpp/logger/stream.h index e57fcf55fd..feb240afcb 100644 --- a/library/cpp/logger/stream.h +++ b/library/cpp/logger/stream.h @@ -1,5 +1,5 @@ #pragma once - + #include "backend.h" class IOutputStream; diff --git a/library/cpp/logger/system.h b/library/cpp/logger/system.h index 5eeca896c6..b8c60b3023 100644 --- a/library/cpp/logger/system.h +++ b/library/cpp/logger/system.h @@ -1,5 +1,5 @@ #pragma once - + #include "log.h" #include "backend.h" #include "priority.h" diff --git a/library/cpp/logger/thread.h b/library/cpp/logger/thread.h index 1ea222a0e6..65f7a88e87 100644 --- a/library/cpp/logger/thread.h +++ b/library/cpp/logger/thread.h @@ -1,5 +1,5 @@ #pragma once - + #include "backend.h" #include <util/generic/ptr.h> diff --git a/library/cpp/messagebus/coreconn.cpp b/library/cpp/messagebus/coreconn.cpp index d25f48930e..d9411bb5db 100644 --- a/library/cpp/messagebus/coreconn.cpp +++ b/library/cpp/messagebus/coreconn.cpp @@ -8,7 +8,7 @@ #include <util/string/util.h> #include <util/system/thread.h> -namespace NBus { +namespace NBus { TBusInstant Now() { return millisec(); } diff --git a/library/cpp/messagebus/coreconn.h b/library/cpp/messagebus/coreconn.h index 7547c9aba6..fca228d82e 100644 --- a/library/cpp/messagebus/coreconn.h +++ b/library/cpp/messagebus/coreconn.h @@ -27,10 +27,10 @@ #include <deque> #include <utility> -#ifdef NO_ERROR +#ifdef NO_ERROR #undef NO_ERROR -#endif - +#endif + #define BUS_WORKER_CONDVAR //#define BUS_WORKER_MIXED @@ -63,5 +63,5 @@ namespace NBus { POLL_READ, POLL_WRITE }; - + } diff --git a/library/cpp/messagebus/handler.cpp b/library/cpp/messagebus/handler.cpp index 3ad336c91d..333bd52934 100644 --- a/library/cpp/messagebus/handler.cpp +++ b/library/cpp/messagebus/handler.cpp @@ -1,7 +1,7 @@ #include "handler.h" #include "remote_server_connection.h" -#include "ybus.h" +#include "ybus.h" using namespace NBus; using namespace NBus::NPrivate; diff --git a/library/cpp/messagebus/locator.cpp b/library/cpp/messagebus/locator.cpp index 11716e3cd5..e38a35c426 100644 --- a/library/cpp/messagebus/locator.cpp +++ b/library/cpp/messagebus/locator.cpp @@ -9,7 +9,7 @@ #include <util/generic/hash_set.h> #include <util/system/hostname.h> -namespace NBus { +namespace NBus { using namespace NAddr; static TIpPort GetAddrPort(const IRemoteAddr& addr) { @@ -360,7 +360,7 @@ namespace NBus { } int TBusLocator::LocateKeys(TBusService service, TBusKeyVec& keys, bool onlyLocal) { - TGuard<TMutex> G(Lock); + TGuard<TMutex> G(Lock); Y_VERIFY(keys.empty(), "Non empty keys"); TServiceId serviceId = GetServiceId(service); @@ -374,7 +374,7 @@ namespace NBus { continue; } keys.push_back(std::make_pair(item.Start, item.End)); - } + } return (int)keys.size(); } diff --git a/library/cpp/messagebus/message.cpp b/library/cpp/messagebus/message.cpp index 0fba16bc68..bfa7ed8e9b 100644 --- a/library/cpp/messagebus/message.cpp +++ b/library/cpp/messagebus/message.cpp @@ -9,7 +9,7 @@ using namespace NBus; -namespace NBus { +namespace NBus { using namespace NBus::NPrivate; TBusIdentity::TBusIdentity() @@ -156,7 +156,7 @@ namespace NBus { ReplyTo = data.Connection->PeerAddrSocketAddr; SetCompressed(compressed || IsCompressedResponse()); } - + void TBusMessage::SetCompressed(bool v) { if (v) { GetHeader()->FlagsInternal |= MESSAGE_COMPRESS_INTERNAL; diff --git a/library/cpp/messagebus/messqueue.cpp b/library/cpp/messagebus/messqueue.cpp index 5de2c1a12c..3474d62705 100644 --- a/library/cpp/messagebus/messqueue.cpp +++ b/library/cpp/messagebus/messqueue.cpp @@ -24,17 +24,17 @@ TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, TBus TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, const char* name) { return CreateMessageQueue(config, new TBusLocator, name); -} - +} + TBusMessageQueuePtr NBus::CreateMessageQueue(TExecutorPtr executor, const char* name) { return CreateMessageQueue(TBusQueueConfig(), executor, new TBusLocator, name); } TBusMessageQueuePtr NBus::CreateMessageQueue(const char* name) { - TBusQueueConfig config; + TBusQueueConfig config; return CreateMessageQueue(config, name); -} - +} + namespace { TBusQueueConfig QueueConfigFillDefaults(const TBusQueueConfig& orig, const TString& name) { TBusQueueConfig patched = orig; @@ -148,8 +148,8 @@ TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IB } catch (...) { Y_FAIL("create destination failure: %s", CurrentExceptionMessage().c_str()); } -} - +} + TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, const TVector<TBindResult>& bindTo, const TString& name) { TRemoteServerSessionPtr session(new TRemoteServerSession(this, proto, handler, config, name)); try { @@ -179,15 +179,15 @@ void TBusMessageQueue::Destroy(TBusSession* session) { void TBusMessageQueue::DestroyAllSessions() { TList<TIntrusivePtr<TBusSessionImpl>> sessions; - { - TGuard<TMutex> scope(Lock); + { + TGuard<TMutex> scope(Lock); sessions = Sessions; - } - + } + for (auto& session : sessions) { Y_VERIFY(session->IsDown(), "Session must be shut down prior to queue shutdown"); } -} +} void TBusMessageQueue::Schedule(IScheduleItemAutoPtr i) { Scheduler.Schedule(i); diff --git a/library/cpp/messagebus/network.cpp b/library/cpp/messagebus/network.cpp index 214fbb5f1f..304bedae5a 100644 --- a/library/cpp/messagebus/network.cpp +++ b/library/cpp/messagebus/network.cpp @@ -1,8 +1,8 @@ -#include "network.h" - +#include "network.h" + #include <util/generic/maybe.h> #include <util/generic/ptr.h> -#include <util/network/init.h> +#include <util/network/init.h> #include <util/network/socket.h> #include <util/system/platform.h> @@ -19,7 +19,7 @@ namespace { } int one = 1; - int r1 = SetSockOpt(fd, SOL_SOCKET, SO_REUSEADDR, one); + int r1 = SetSockOpt(fd, SOL_SOCKET, SO_REUSEADDR, one); if (r1 < 0) { ythrow TSystemError() << "failed to setsockopt SO_REUSEADDR"; } diff --git a/library/cpp/messagebus/oldmodule/module.cpp b/library/cpp/messagebus/oldmodule/module.cpp index 3dc9c80a94..24bd778799 100644 --- a/library/cpp/messagebus/oldmodule/module.cpp +++ b/library/cpp/messagebus/oldmodule/module.cpp @@ -60,7 +60,7 @@ namespace { } -namespace NBus { +namespace NBus { namespace NPrivate { class TJobStorage { }; @@ -327,11 +327,11 @@ namespace NBus { { Handler = TJobHandler(&TBusModule::Start); } - + TBusJob::~TBusJob() { Y_ASSERT(Pending.size() == 0); //Y_ASSERT(SleepUntil == 0); - + ClearAllMessageStates(); } @@ -618,7 +618,7 @@ namespace NBus { } ////////////////////////////////////////////////////////////////////// - + void TBusModuleImpl::CancelJob(TBusJob* job, EMessageStatus status) { TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for CancelJob"); if (job) { @@ -636,8 +636,8 @@ namespace NBus { strReturn += "TODO\n"; } return strReturn; - } - + } + TBusModuleConfig::TBusModuleConfig() : StarterMaxInFlight(1000) { @@ -652,10 +652,10 @@ namespace NBus { : Impl(new TBusModuleImpl(this, name)) { } - + TBusModule::~TBusModule() { } - + const char* TBusModule::GetName() const { return Impl->Name; } @@ -680,7 +680,7 @@ namespace NBus { bool TBusModule::Shutdown() { Impl->Shutdown(); - + return true; } @@ -688,16 +688,16 @@ namespace NBus { TBusJob* job = new TBusJob(this, message); return job; } - + /** -Example for external session creation: - -TBusSession* TMyModule::CreateExtSession(TBusMessageQueue& queue) { - TBusSession* session = CreateDefaultDestination(queue, &ExternalProto, ExternalConfig); - session->RegisterService(hostname, begin, end); - return session; -*/ - +Example for external session creation: + +TBusSession* TMyModule::CreateExtSession(TBusMessageQueue& queue) { + TBusSession* session = CreateDefaultDestination(queue, &ExternalProto, ExternalConfig); + session->RegisterService(hostname, begin, end); + return session; +*/ + bool TBusModule::CreatePrivateSessions(TBusMessageQueue* queue) { Impl->Queue = queue; return true; @@ -782,11 +782,11 @@ void TBusModuleImpl::DestroyJob(TJobRunner* job) { ShutdownCondVar.BroadCast(); } } - } + } job->JobStorageIterator = TList<TJobRunner*>::iterator(); } - + void TBusModuleImpl::OnMessageReceived(TAutoPtr<TBusMessage> msg0, TOnMessageContext& context) { TBusMessage* msg = !!msg0 ? msg0.Get() : context.GetMessage(); Y_VERIFY(!!msg); @@ -797,7 +797,7 @@ void TBusModuleImpl::OnMessageReceived(TAutoPtr<TBusMessage> msg0, TOnMessageCon SetJob(jobRunner->Job->Message, jobRunner.Get()); AtomicIncrement(JobCount); - + AddJob(jobRunner.Get()); jobRunner.Release()->Schedule(); @@ -858,14 +858,14 @@ void TModuleClientHandler::OnReply(TAutoPtr<TBusMessage> req, TAutoPtr<TBusMessa job->EnqueueAndSchedule(TJobResponseMessage(req.Release(), resp.Release(), MESSAGE_OK)); job->UnRef(); } - + void TModuleClientHandler::OnMessageSentOneWay(TAutoPtr<TBusMessage> req) { TJobRunner* job = GetJob(req.Get()); Y_ASSERT(job); Y_ASSERT(job->Job->Message != req.Get()); job->EnqueueAndSchedule(TJobResponseMessage(req.Release(), nullptr, MESSAGE_OK)); job->UnRef(); -} +} void TModuleClientHandler::OnError(TAutoPtr<TBusMessage> msg, EMessageStatus status) { TJobRunner* job = GetJob(msg.Get()); diff --git a/library/cpp/messagebus/oldmodule/module.h b/library/cpp/messagebus/oldmodule/module.h index dc3b6377f0..8d1c4a5d52 100644 --- a/library/cpp/messagebus/oldmodule/module.h +++ b/library/cpp/messagebus/oldmodule/module.h @@ -9,7 +9,7 @@ /// NBus::TBusSession. /// To implement the module some virtual functions needs to be overridden: - + /// NBus::TBusModule::CreateExtSession() creates and registers an /// external session that receives incoming messages as input for module /// processing. @@ -18,13 +18,13 @@ /// NBus::TBusJob is somewhat similar to a thread, it maintains all the state /// during processing of one incoming message. Default implementation of /// NBus::TBusJob will maintain all send and received messages during -/// lifetime of this job. Each message, status and reply can be found +/// lifetime of this job. Each message, status and reply can be found /// within NBus::TJobState using NBus::TBusJob::GetState(). If your module /// needs to maintain an additional information during lifetime of the job /// you can derive your own class from NBus::TBusJob and override job -/// factory method NBus::IJobFactory::CreateJobInstance() to create your instances. +/// factory method NBus::IJobFactory::CreateJobInstance() to create your instances. -/// Processing of a given message starts with a call to NBus::TBusModule::Start() +/// Processing of a given message starts with a call to NBus::TBusModule::Start() /// handler that should be overridden in the module implementation. Within /// the callback handler module can perform any computation and access any /// datastore tables that it needs. The handler can also access any module @@ -33,7 +33,7 @@ /// Handler should use NBus::TBusJob::Send() to send messages to other client /// sessions and it can use NBus::TBusJob::Reply() to send reply to the main -/// job message. When handler is done, it returns the pointer to the next handler to call +/// job message. When handler is done, it returns the pointer to the next handler to call /// when all pending messages have cleared. If handler /// returns pointer to itself the module will reschedule execution of this handler /// for a later time. This should be done in case NBus::TBusJob::Send() returns diff --git a/library/cpp/messagebus/oldmodule/startsession.cpp b/library/cpp/messagebus/oldmodule/startsession.cpp index 0827972d88..7c38801d62 100644 --- a/library/cpp/messagebus/oldmodule/startsession.cpp +++ b/library/cpp/messagebus/oldmodule/startsession.cpp @@ -18,7 +18,7 @@ #include <library/cpp/messagebus/ybus.h> -namespace NBus { +namespace NBus { void* TBusStarter::_starter(void* data) { TBusStarter* pThis = static_cast<TBusStarter*>(data); pThis->Starter(); @@ -61,5 +61,5 @@ namespace NBus { } } } - -} + +} diff --git a/library/cpp/messagebus/protobuf/ybusbuf.h b/library/cpp/messagebus/protobuf/ybusbuf.h index 1fed492d5c..57b4267ea5 100644 --- a/library/cpp/messagebus/protobuf/ybusbuf.h +++ b/library/cpp/messagebus/protobuf/ybusbuf.h @@ -226,8 +226,8 @@ namespace NBus { /// serialized protocol specific data into TBusData void Serialize(const TBusMessage* mess, TBuffer& data) override; - + TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; }; - + } diff --git a/library/cpp/messagebus/scheduler/scheduler.cpp b/library/cpp/messagebus/scheduler/scheduler.cpp index dd9767c25a..5a5fe52894 100644 --- a/library/cpp/messagebus/scheduler/scheduler.cpp +++ b/library/cpp/messagebus/scheduler/scheduler.cpp @@ -4,7 +4,7 @@ #include <util/generic/algorithm.h> #include <util/generic/yexception.h> -//#include "dummy_debugger.h" +//#include "dummy_debugger.h" using namespace NBus; using namespace NBus::NPrivate; diff --git a/library/cpp/messagebus/session.cpp b/library/cpp/messagebus/session.cpp index 69b6825bb8..46a7ece6a8 100644 --- a/library/cpp/messagebus/session.cpp +++ b/library/cpp/messagebus/session.cpp @@ -4,7 +4,7 @@ using namespace NBus; -namespace NBus { +namespace NBus { TBusSession::TBusSession() { } diff --git a/library/cpp/messagebus/synchandler.cpp b/library/cpp/messagebus/synchandler.cpp index 30c2cc3806..8e891d66b3 100644 --- a/library/cpp/messagebus/synchandler.cpp +++ b/library/cpp/messagebus/synchandler.cpp @@ -142,7 +142,7 @@ namespace NBus { // deletion of message and reply is a job of application. pMessage->Data = nullptr; - + return reply; } }; diff --git a/library/cpp/messagebus/test/perftest/perftest.cpp b/library/cpp/messagebus/test/perftest/perftest.cpp index f7f5e88086..8489319278 100644 --- a/library/cpp/messagebus/test/perftest/perftest.cpp +++ b/library/cpp/messagebus/test/perftest/perftest.cpp @@ -593,12 +593,12 @@ int main(int argc, char* argv[]) { /* unix foo */ setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); - Umask(0); + Umask(0); SetAsyncSignalHandler(SIGINT, stopsignal); SetAsyncSignalHandler(SIGTERM, stopsignal); -#ifndef _win_ +#ifndef _win_ SetAsyncSignalHandler(SIGUSR1, stopsignal); -#endif +#endif signal(SIGPIPE, SIG_IGN); NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); @@ -641,7 +641,7 @@ int main(int argc, char* argv[]) { TIntrusivePtr<TBusWww> www(new TBusWww); ServerAddresses = ParseNodes(TheConfig->Nodes); - + if (TheConfig->ServerPort) { if (TheConfig->ServerUseModules) { ServerUsingModule = new TPerftestUsingModule(); diff --git a/library/cpp/messagebus/test/ut/one_way_ut.cpp b/library/cpp/messagebus/test/ut/one_way_ut.cpp index 16065289ad..9c21227e2b 100644 --- a/library/cpp/messagebus/test/ut/one_way_ut.cpp +++ b/library/cpp/messagebus/test/ut/one_way_ut.cpp @@ -55,7 +55,7 @@ struct NullClient : TBusClientHandlerError { UNIT_ASSERT(serverAddr.GetPort() > 0); /// create or get instance of message queue, need one per application - Queue = CreateMessageQueue(); + Queue = CreateMessageQueue(); /// register source/client session Session = TBusClientSession::Create(&Proto, this, sessionConfig, Queue); @@ -99,7 +99,7 @@ public: NumMessages = 0; /// create or get instance of single message queue, need one for application - Queue = CreateMessageQueue(); + Queue = CreateMessageQueue(); /// register destination session TBusServerSessionConfig sessionConfig; diff --git a/library/cpp/messagebus/ya.make b/library/cpp/messagebus/ya.make index e8b6701eed..e13cf06dea 100644 --- a/library/cpp/messagebus/ya.make +++ b/library/cpp/messagebus/ya.make @@ -1,4 +1,4 @@ -LIBRARY() +LIBRARY() OWNER(g:messagebus) diff --git a/library/cpp/messagebus/ybus.h b/library/cpp/messagebus/ybus.h index 2393b840ac..de21ad8521 100644 --- a/library/cpp/messagebus/ybus.h +++ b/library/cpp/messagebus/ybus.h @@ -74,10 +74,10 @@ namespace NBus { int GetPort() const { return ServicePort; } - + virtual ~TBusProtocol() { } - + /// \brief serialized protocol specific data into TBusData /// \note buffer passed to the function (data) is not empty, use append functions virtual void Serialize(const TBusMessage* mess, TBuffer& data) = 0; @@ -175,7 +175,7 @@ namespace NBus { TBusLocator* GetLocator() const { return Locator.Get(); } - + TBusClientSessionPtr CreateSource(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, const TString& name = ""); TBusSyncClientSessionPtr CreateSyncSource(TBusProtocol* proto, const TBusClientSessionConfig& config, bool needReply = true, const TString& name = ""); TBusServerSessionPtr CreateDestination(TBusProtocol* proto, IBusServerHandler* hander, const TBusServerSessionConfig& config, const TString& name = ""); diff --git a/library/cpp/mime/types/mime.cpp b/library/cpp/mime/types/mime.cpp index 2dfa73ef2b..706d776b24 100644 --- a/library/cpp/mime/types/mime.cpp +++ b/library/cpp/mime/types/mime.cpp @@ -5,7 +5,7 @@ #include <util/generic/strbuf.h> #include <util/generic/singleton.h> #include <util/generic/yexception.h> - + #include <cctype> /* @@ -175,7 +175,7 @@ const char* mimetypeByExt(const char* fname, const char* check_ext) { size_t i; ext_p++; for (i = 0; i < TMimeTypes::MAX_EXT_LEN && ext_p[i]; i++) - ext[i] = (char)tolower(ext_p[i]); + ext[i] = (char)tolower(ext_p[i]); ext[i] = 0; if (check_ext != nullptr) { diff --git a/library/cpp/mime/types/mime.h b/library/cpp/mime/types/mime.h index ff30d24e43..05da389ea9 100644 --- a/library/cpp/mime/types/mime.h +++ b/library/cpp/mime/types/mime.h @@ -3,9 +3,9 @@ #include <util/system/defaults.h> #include <util/generic/strbuf.h> -#include <cstring> +#include <cstring> -enum MimeTypes { +enum MimeTypes { MIME_UNKNOWN = 0, MIME_TEXT = 1, MIME_HTML = 2, @@ -58,9 +58,9 @@ enum MimeTypes { MIME_WOFF2 = 44, MIME_TTF = 45, MIME_WEBMANIFEST = 46, - MIME_MAX -}; - + MIME_MAX +}; + extern const char* MimeNames[MIME_MAX]; const char* mimetypeByExt(const char* fname, const char* check_ext = nullptr); diff --git a/library/cpp/monlib/service/service.h b/library/cpp/monlib/service/service.h index d8318a5249..2f66dddaf8 100644 --- a/library/cpp/monlib/service/service.h +++ b/library/cpp/monlib/service/service.h @@ -11,7 +11,7 @@ #include <functional> -struct TMonitor; +struct TMonitor; namespace NMonitoring { struct IHttpRequest { diff --git a/library/cpp/on_disk/chunks/chunked_helpers.h b/library/cpp/on_disk/chunks/chunked_helpers.h index 5395f8c9b4..5fa96afdca 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.h +++ b/library/cpp/on_disk/chunks/chunked_helpers.h @@ -269,7 +269,7 @@ public: bits = 0; } WriteBin<ui16>(&out, bits); - WriteBin<ui32>(&out, (ui32)Data.size()); + WriteBin<ui32>(&out, (ui32)Data.size()); const ui32 nBuckets = ui32(1) << bits; TData2 data2(nBuckets); @@ -281,8 +281,8 @@ public: ui32 offset = 0; for (ui32 i = 0; i < nBuckets; ++i) { intervals[i].Offset = offset; - intervals[i].Length = (ui32)data2[i].size(); - offset += (ui32)data2[i].size(); + intervals[i].Length = (ui32)data2[i].size(); + offset += (ui32)data2[i].size(); } #ifndef NDEBUG for (ui32 i = 0; i < nBuckets; ++i) { diff --git a/library/cpp/on_disk/chunks/chunks_ut.cpp b/library/cpp/on_disk/chunks/chunks_ut.cpp index bcd6801cb3..f727647f7f 100644 --- a/library/cpp/on_disk/chunks/chunks_ut.cpp +++ b/library/cpp/on_disk/chunks/chunks_ut.cpp @@ -90,8 +90,8 @@ public: writer.Save(stream); } { - TBlob temp = TBlob::FromStreamSingleThreaded(stream); - TGeneralVector<ui32> reader(temp); + TBlob temp = TBlob::FromStreamSingleThreaded(stream); + TGeneralVector<ui32> reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { ui32 value; @@ -112,7 +112,7 @@ public: writer.Save(stream); } { - TBlob temp = TBlob::FromStreamSingleThreaded(stream); + TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<TString> reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { @@ -135,8 +135,8 @@ public: writer.Save(stream); } { - TBlob temp = TBlob::FromStreamSingleThreaded(stream); - TGeneralVector<TItem> reader(temp); + TBlob temp = TBlob::FromStreamSingleThreaded(stream); + TGeneralVector<TItem> reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); TItem value; @@ -163,8 +163,8 @@ public: writer.Save(stream); } { - TBlob temp = TBlob::FromStreamSingleThreaded(stream); - TGeneralVector<int*> reader(temp); + TBlob temp = TBlob::FromStreamSingleThreaded(stream); + TGeneralVector<int*> reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { int* value; @@ -186,8 +186,8 @@ public: writer.Save(stream); } { - TBlob temp = TBlob::FromStreamSingleThreaded(stream); - TGeneralVector<TItem> reader(temp); + TBlob temp = TBlob::FromStreamSingleThreaded(stream); + TGeneralVector<TItem> reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); for (size_t i = 0; i < N; ++i) { TItem value; @@ -214,7 +214,7 @@ public: } { - TBlob fIn = TBlob::FromFileSingleThreaded(FILENAME); + TBlob fIn = TBlob::FromFileSingleThreaded(FILENAME); TStringsVector vct(GetBlock(fIn, 0)); UNIT_ASSERT_EQUAL(vct.Get(0), ""); UNIT_ASSERT_EQUAL(vct.Get(1), "test"); @@ -250,7 +250,7 @@ public: } { - TBlob mf = TBlob::FromFileSingleThreaded(filename); + TBlob mf = TBlob::FromFileSingleThreaded(filename); TNamedChunkedDataReader reader(mf); UNIT_ASSERT(reader.GetBlocksCount() == 3); @@ -297,7 +297,7 @@ private: writer.WriteFooter(); } { - TBlob blob = TBlob::FromBufferSingleThreaded(buffer); + TBlob blob = TBlob::FromBufferSingleThreaded(buffer); TChunkedDataReader data(blob); // printf("%d\n", (int)data.GetBlockLen(3)); UNIT_ASSERT_EQUAL(4, data.GetBlockLen(0)); @@ -317,7 +317,7 @@ private: writer.WriteFooter(); } { - TBlob blob = TBlob::FromBufferSingleThreaded(buffer); + TBlob blob = TBlob::FromBufferSingleThreaded(buffer); TChunkedDataReader data(blob); // printf("%d\n", (int)data.GetBlockLen(1)); UNIT_ASSERT_EQUAL(0, data.GetBlockLen(0)); diff --git a/library/cpp/packers/packers.h b/library/cpp/packers/packers.h index 20248defae..1bde1b59aa 100644 --- a/library/cpp/packers/packers.h +++ b/library/cpp/packers/packers.h @@ -64,27 +64,27 @@ namespace NPackers { } } - namespace NImpl { - template <class T, bool isSigned> - struct TConvertImpl { - static inline ui64 Convert(const T& data); - }; - - template <class T> - struct TConvertImpl<T, true> { - static inline ui64 Convert(const T& data) { - return ConvertIntegral<i64>(static_cast<i64>(data)); - } - }; - - template <class T> - struct TConvertImpl<T, false> { - static inline ui64 Convert(const T& data) { - return data; - } - }; - } - + namespace NImpl { + template <class T, bool isSigned> + struct TConvertImpl { + static inline ui64 Convert(const T& data); + }; + + template <class T> + struct TConvertImpl<T, true> { + static inline ui64 Convert(const T& data) { + return ConvertIntegral<i64>(static_cast<i64>(data)); + } + }; + + template <class T> + struct TConvertImpl<T, false> { + static inline ui64 Convert(const T& data) { + return data; + } + }; + } + template <class T> inline ui64 ConvertIntegral(const T& data) { static_assert(std::is_integral<T>::value, "T must be integral type"); @@ -149,33 +149,33 @@ namespace NPackers { return SkipTable[(ui8)*p]; } - namespace NImpl { - template <class T, bool isSigned> - struct TUnpackLeafImpl { + namespace NImpl { + template <class T, bool isSigned> + struct TUnpackLeafImpl { inline void UnpackLeaf(const char* p, T& t) const; - }; - template <class T> - struct TUnpackLeafImpl<T, true> { + }; + template <class T> + struct TUnpackLeafImpl<T, true> { inline void UnpackLeaf(const char* p, T& t) const { ui64 val; TIntegralPacker<ui64>().UnpackLeaf(p, val); - if (val & 1) { + if (val & 1) { t = -1 * static_cast<i64>(val >> 1); - } else { + } else { t = static_cast<T>(val >> 1); - } - } - }; - template <class T> - struct TUnpackLeafImpl<T, false> { + } + } + }; + template <class T> + struct TUnpackLeafImpl<T, false> { inline void UnpackLeaf(const char* p, T& t) const { ui64 tmp; TIntegralPacker<ui64>().UnpackLeaf(p, tmp); t = static_cast<T>(tmp); - } - }; - } - + } + }; + } + template <class T> inline void TIntegralPacker<T>::UnpackLeaf(const char* p, T& t) const { NImpl::TUnpackLeafImpl<T, std::is_signed<T>::value>().UnpackLeaf(p, t); diff --git a/library/cpp/regex/pcre/regexp.cpp b/library/cpp/regex/pcre/regexp.cpp index e7c2dfa047..575c09cee4 100644 --- a/library/cpp/regex/pcre/regexp.cpp +++ b/library/cpp/regex/pcre/regexp.cpp @@ -4,7 +4,7 @@ #include <util/string/ascii.h> #include <util/system/defaults.h> -#include <cstdlib> +#include <cstdlib> #include <util/generic/noncopyable.h> class TGlobalImpl : TNonCopyable { @@ -24,7 +24,7 @@ private: }; private: - void CopyResults(int count) { + void CopyResults(int count) { for (int i = 0; i < count; i++) { Pmatch[MatchPos].rm_so = MatchBuf[2 * i]; Pmatch[MatchPos].rm_eo = MatchBuf[2 * i + 1]; @@ -35,7 +35,7 @@ private: } } - int DoPcreExec(int opts) { + int DoPcreExec(int opts) { int rc = pcre_exec( PregComp, /* the compiled pattern */ nullptr, /* no extra data - we didn't study the pattern */ @@ -89,7 +89,7 @@ public: StrLen = strlen(Str); } - int ExecGlobal() { + int ExecGlobal() { StartOffset = 0; int rc = DoPcreExec(Options); diff --git a/library/cpp/regex/pire/regexp.h b/library/cpp/regex/pire/regexp.h index 215263f8f3..94bba4064b 100644 --- a/library/cpp/regex/pire/regexp.h +++ b/library/cpp/regex/pire/regexp.h @@ -11,7 +11,7 @@ #include <util/generic/yexception.h> namespace NRegExp { - struct TMatcher; + struct TMatcher; struct TFsmBase { struct TOptions { diff --git a/library/cpp/streams/bzip2/bzip2_ut.cpp b/library/cpp/streams/bzip2/bzip2_ut.cpp index 5ea06ac019..69a98f296c 100644 --- a/library/cpp/streams/bzip2/bzip2_ut.cpp +++ b/library/cpp/streams/bzip2/bzip2_ut.cpp @@ -2,7 +2,7 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/system/tempfile.h> #define ZDATA "./zdata" diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp index c046103b55..6876f070fc 100644 --- a/library/cpp/streams/lz/lz_ut.cpp +++ b/library/cpp/streams/lz/lz_ut.cpp @@ -3,7 +3,7 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/resource/resource.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/generic/vector.h> #include <util/system/tempfile.h> #include <util/generic/singleton.h> diff --git a/library/cpp/string_utils/base64/base64.cpp b/library/cpp/string_utils/base64/base64.cpp index ab1078e889..05c201f0de 100644 --- a/library/cpp/string_utils/base64/base64.cpp +++ b/library/cpp/string_utils/base64/base64.cpp @@ -89,7 +89,7 @@ static const char base64_bkw[] = { '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0'}; - + static_assert(Y_ARRAY_SIZE(base64_bkw) == 256, "wrong size"); // Base64 for url encoding, RFC3548 @@ -133,13 +133,13 @@ static inline char* Base64EncodeImpl(char* outstr, const unsigned char* instr, s } else { *outstr++ = base64_etab[GetBase64EncodedIndex1(instr[idx], '\0')]; *outstr++ = pad; - } + } *outstr++ = pad; - } + } *outstr = 0; - return outstr; -} + return outstr; +} static char* Base64EncodePlain(char* outstr, const unsigned char* instr, size_t len) { return Base64EncodeImpl<false>(outstr, instr, len); diff --git a/library/cpp/string_utils/quote/quote.cpp b/library/cpp/string_utils/quote/quote.cpp index 1288c8c4ec..e523350b80 100644 --- a/library/cpp/string_utils/quote/quote.cpp +++ b/library/cpp/string_utils/quote/quote.cpp @@ -112,12 +112,12 @@ static inline It1 Escape(It1 to, It2 from, It3 end, const bool* escape_map = cha *to++ = d2x((unsigned char)*from & 0xF); } else { *to++ = (*from == ' ' ? '+' : *from); - } + } ++from; - } + } - *to = 0; + *to = 0; return to; } @@ -220,7 +220,7 @@ char* CGIUnescape(char* to, const char* from) { char* CGIUnescape(char* to, const char* from, size_t len) { return Unescape(to, from, from + len, TFromHexLenLimited(from + len)); -} +} void CGIUnescape(TString& url) { if (url.empty()) { @@ -234,7 +234,7 @@ void CGIUnescape(TString& url) { url = CGIUnescapeRet(url); } } - + TString CGIUnescapeRet(const TStringBuf from) { TString to; to.ReserveAndResize(CgiUnescapeBufLen(from.size())); @@ -249,13 +249,13 @@ char* UrlUnescape(char* to, TStringBuf from) { if ('%' == ch && 2 <= from.length()) ch = TFromHexZeroTerm::x2c(from); *to++ = ch; - } + } - *to = 0; + *to = 0; return to; -} - +} + void UrlUnescape(TString& url) { if (url.empty()) { return; @@ -267,8 +267,8 @@ void UrlUnescape(TString& url) { } else { url = UrlUnescapeRet(url); } -} - +} + TString UrlUnescapeRet(const TStringBuf from) { TString to; to.ReserveAndResize(CgiUnescapeBufLen(from.size())); @@ -288,20 +288,20 @@ char* UrlEscape(char* to, const char* from, bool forceEscape) { *to++ = d2x((unsigned char)*from >> 4); *to++ = d2x((unsigned char)*from & 0xF); } else - *to++ = *from; + *to++ = *from; ++from; - } + } - *to = 0; + *to = 0; return to; -} - +} + void UrlEscape(TString& url, bool forceEscape) { TTempBuf tempBuf(CgiEscapeBufLen(url.size())); char* to = tempBuf.Data(); url.AssignNoAlias(to, UrlEscape(to, url.data(), forceEscape)); -} +} TString UrlEscapeRet(const TStringBuf from, bool forceEscape) { TString to; diff --git a/library/cpp/string_utils/url/url.cpp b/library/cpp/string_utils/url/url.cpp index 32179ead5b..85f4ac5d69 100644 --- a/library/cpp/string_utils/url/url.cpp +++ b/library/cpp/string_utils/url/url.cpp @@ -53,15 +53,15 @@ namespace { return 0; } - template <typename T> + template <typename T> inline T CutHttpPrefixImpl(const T& url, bool ignorehttps) { size_t prefixSize = GetHttpPrefixSizeImpl<typename T::char_type>(url.data(), TKnownSize(url.size()), ignorehttps); if (prefixSize) return url.substr(prefixSize); - return url; - } -} - + return url; + } +} + namespace NUrl { TSplitUrlToHostAndPathResult SplitUrlToHostAndPath(const TStringBuf url) { @@ -75,8 +75,8 @@ namespace NUrl { size_t GetHttpPrefixSize(const char* url, bool ignorehttps) noexcept { return GetHttpPrefixSizeImpl<char>(url, TUncheckedSize(), ignorehttps); -} - +} + size_t GetHttpPrefixSize(const wchar16* url, bool ignorehttps) noexcept { return GetHttpPrefixSizeImpl<wchar16>(url, TUncheckedSize(), ignorehttps); } @@ -95,8 +95,8 @@ TStringBuf CutHttpPrefix(const TStringBuf url, bool ignorehttps) noexcept { TWtringBuf CutHttpPrefix(const TWtringBuf url, bool ignorehttps) noexcept { return CutHttpPrefixImpl(url, ignorehttps); -} - +} + size_t GetSchemePrefixSize(const TStringBuf url) noexcept { struct TDelim: public str_spn { inline TDelim() @@ -144,8 +144,8 @@ static inline TStringBuf GetHostAndPortImpl(const TStringBuf url) { } return urlNoScheme; -} - +} + TStringBuf GetHost(const TStringBuf url) noexcept { return GetHostAndPortImpl<false>(url); } @@ -279,10 +279,10 @@ TStringBuf GetZone(const TStringBuf host) noexcept { TStringBuf CutWWWPrefix(const TStringBuf url) noexcept { if (url.size() >= 4 && url[3] == '.' && !strnicmp(url.data(), "www", 3)) - return url.substr(4); - return url; -} - + return url.substr(4); + return url; +} + TStringBuf CutWWWNumberedPrefix(const TStringBuf url) noexcept { auto it = url.begin(); @@ -334,44 +334,44 @@ TString AddSchemePrefix(const TString& url, TStringBuf scheme) { return TString::Join(scheme, TStringBuf("://"), url); } -#define X(c) (c >= 'A' ? ((c & 0xdf) - 'A') + 10 : (c - '0')) - -static inline int x2c(unsigned char* x) { +#define X(c) (c >= 'A' ? ((c & 0xdf) - 'A') + 10 : (c - '0')) + +static inline int x2c(unsigned char* x) { if (!IsAsciiHex(x[0]) || !IsAsciiHex(x[1])) - return -1; - return X(x[0]) * 16 + X(x[1]); -} - + return -1; + return X(x[0]) * 16 + X(x[1]); +} + #undef X static inline int Unescape(char* str) { char *to, *from; - int dlen = 0; + int dlen = 0; if ((str = strchr(str, '%')) == nullptr) - return dlen; - for (to = str, from = str; *from; from++, to++) { - if ((*to = *from) == '%') { + return dlen; + for (to = str, from = str; *from; from++, to++) { + if ((*to = *from) == '%') { int c = x2c((unsigned char*)from + 1); - *to = char((c > 0) ? c : '0'); - from += 2; - dlen += 2; - } - } + *to = char((c > 0) ? c : '0'); + from += 2; + dlen += 2; + } + } *to = 0; /* terminate it at the new length */ - return dlen; -} - + return dlen; +} + size_t NormalizeUrlName(char* dest, const TStringBuf source, size_t dest_size) { if (source.empty() || source[0] == '?') - return strlcpy(dest, "/", dest_size); + return strlcpy(dest, "/", dest_size); size_t len = Min(dest_size - 1, source.length()); memcpy(dest, source.data(), len); dest[len] = 0; - len -= Unescape(dest); - strlwr(dest); - return len; -} - + len -= Unescape(dest); + strlwr(dest); + return len; +} + size_t NormalizeHostName(char* dest, const TStringBuf source, size_t dest_size, ui16 defport) { size_t len = Min(dest_size - 1, source.length()); memcpy(dest, source.data(), len); @@ -382,11 +382,11 @@ size_t NormalizeHostName(char* dest, const TStringBuf source, size_t dest_size, char* ptr = strstr(dest, buf); if (ptr && ptr[buflen] == 0) { len -= buflen; - *ptr = 0; - } - strlwr(dest); - return len; -} + *ptr = 0; + } + strlwr(dest); + return len; +} TStringBuf RemoveFinalSlash(TStringBuf str) noexcept { if (str.EndsWith('/')) { diff --git a/library/cpp/string_utils/url/url.h b/library/cpp/string_utils/url/url.h index be90c4ff8a..84137ccc57 100644 --- a/library/cpp/string_utils/url/url.h +++ b/library/cpp/string_utils/url/url.h @@ -42,17 +42,17 @@ size_t GetSchemePrefixSize(const TStringBuf url) noexcept; Y_PURE_FUNCTION TStringBuf GetSchemePrefix(const TStringBuf url) noexcept; -//! removes protocol prefixes 'http://' and 'https://' from given URL -//! @note if URL has no prefix or some other prefix the function does nothing +//! removes protocol prefixes 'http://' and 'https://' from given URL +//! @note if URL has no prefix or some other prefix the function does nothing //! @param url URL from which the prefix should be removed //! @param ignorehttps if true, leaves https:// -//! @return a new URL without protocol prefix +//! @return a new URL without protocol prefix Y_PURE_FUNCTION TStringBuf CutHttpPrefix(const TStringBuf url, bool ignorehttps = false) noexcept; Y_PURE_FUNCTION TWtringBuf CutHttpPrefix(const TWtringBuf url, bool ignorehttps = false) noexcept; - + Y_PURE_FUNCTION TStringBuf CutSchemePrefix(const TStringBuf url) noexcept; @@ -132,7 +132,7 @@ TStringBuf GetPathAndQuery(const TStringBuf url, bool trimFragment = true) noexc */ Y_PURE_FUNCTION TStringBuf GetOnlyHost(const TStringBuf url) noexcept; - + Y_PURE_FUNCTION TStringBuf GetParentDomain(const TStringBuf host, size_t level) noexcept; // ("www.ya.ru", 2) -> "ya.ru" diff --git a/library/cpp/uri/common.cpp b/library/cpp/uri/common.cpp index d3623f8208..05af1e57d1 100644 --- a/library/cpp/uri/common.cpp +++ b/library/cpp/uri/common.cpp @@ -103,8 +103,8 @@ namespace NUri { default: return "Field[Unknown]"; } - } - + } + const char* SchemeKindToString(const TScheme::EKind& t) { const TSchemeInfo& info = TSchemeInfo::Get(t); if (!info.Str.empty()) diff --git a/library/cpp/uri/uri-ru_ut.cpp b/library/cpp/uri/uri-ru_ut.cpp index 0cbc7a9472..ec35a164d2 100644 --- a/library/cpp/uri/uri-ru_ut.cpp +++ b/library/cpp/uri/uri-ru_ut.cpp @@ -2,7 +2,7 @@ #include <library/cpp/charset/recyr.hh> #include <library/cpp/html/entity/htmlentity.h> #include <util/system/maxlen.h> - + namespace NUri { namespace { TString AsWin1251(const TString& s) { @@ -70,7 +70,7 @@ namespace NUri { "images\nil.jpg", "images%0Ail.jpg", "http://caedebaturque.termez.su\r\n/?article=218", "http://caedebaturque.termez.su%0D%0A/?article=218", - + AsKoi8("javascript:window.external.AddFavorite(\'http://www.humor.look.ru/\',\'Злобные Деды Морозы!!!\')"), "javascript:window.external.AddFavorite(\'http://www.humor.look.ru/\',\'%FA%CC%CF%C2%CE%D9%C5%20%E4%C5%C4%D9%20%ED%CF%D2%CF%DA%D9!!!\')", "search.php?search_author=%CB%FE%E4%EC%E8%EB%E0+%C3%F3%F1%E5%E2%E0&showresults=posts&sid=8", "search.php?search_author=%CB%FE%E4%EC%E8%EB%E0+%C3%F3%F1%E5%E2%E0&showresults=posts&sid=8", AsWin1251("/Search/author/?q=Штрибель Х.В."), "/Search/author/?q=%D8%F2%F0%E8%E1%E5%EB%FC%20%D5.%C2.", @@ -108,7 +108,7 @@ namespace NUri { "/a-mp3/stype-1/?search=Э", "/a-mp3/stype-1/?search=%D0%AD", "/a-mp3/stype-1/?search=Ю", "/a-mp3/stype-1/?search=%D0%AE", "/a-mp3/stype-1/?search=Я", "/a-mp3/stype-1/?search=%D0%AF", - + "javascript:emoticon(\":'(\")", "javascript:emoticon(\":\'(\")", "javascript:emoticon(\'>:o\')", "javascript:emoticon(\'>:o\')", "javascript:emoticon(\']:->\')", "javascript:emoticon(\']:->\')", @@ -119,7 +119,7 @@ namespace NUri { ""http://www.fubix.ru"", "\"http://www.fubix.ru\"", AsWin1251("mailto:kampa@ukr.net?subject=Арабский язык"), "mailto:kampa@ukr.net?subject=%C0%F0%E0%E1%F1%EA%E8%E9%20%FF%E7%FB%EA", {}}; - + Y_UNIT_TEST(testHtLinkDecode) { char decodedlink[URL_MAXLEN + 10]; for (int i = 0; links[i]; i += 2) { @@ -127,7 +127,7 @@ namespace NUri { UNIT_ASSERT_VALUES_EQUAL(decodedlink, links[i + 1]); } } - + Y_UNIT_TEST(testRuIDNA) { { #define DEC "\xD7\xE5\xF0\xE5\xEf\xEE\xE2\xE5\xF6.\xF0\xF4" /* "Череповец.рф" in Windows-1251 */ @@ -160,4 +160,4 @@ namespace NUri { } } -} +} diff --git a/library/cpp/uri/uri.cpp b/library/cpp/uri/uri.cpp index 2bb3ceac24..56a9a4e5ef 100644 --- a/library/cpp/uri/uri.cpp +++ b/library/cpp/uri/uri.cpp @@ -616,6 +616,6 @@ namespace NUri { } Y_ASSERT(0); return ""; - } - + } + } diff --git a/library/cpp/uri/uri.h b/library/cpp/uri/uri.h index f6dc8426a2..3b6c19fe4a 100644 --- a/library/cpp/uri/uri.h +++ b/library/cpp/uri/uri.h @@ -534,7 +534,7 @@ namespace NUri { Y_ASSERT(IsValidAbs() && other.IsValidAbs()); return Equal(other, FlagScheme | FlagHostPort); } - + TLinkType Locality(const TUri& other) const { if (IsSameDocument(other)) return LinkIsFragment; @@ -542,11 +542,11 @@ namespace NUri { return LinkIsLocal; return LinkIsGlobal; } - + static IOutputStream& ReEncodeField(IOutputStream& out, const TStringBuf& val, EField fld, long flags = FeaturesEncodeDecode) { return NEncode::TEncoder::ReEncode(out, val, NEncode::TEncodeMapper(flags, fld)); } - + static IOutputStream& ReEncodeToField(IOutputStream& out, const TStringBuf& val, EField srcfld, long srcflags, EField dstfld, long dstflags) { return NEncode::TEncoder::ReEncodeTo(out, val, NEncode::TEncodeMapper(srcflags, srcfld), NEncode::TEncodeToMapper(dstflags, dstfld)); } diff --git a/library/cpp/uri/uri_ut.cpp b/library/cpp/uri/uri_ut.cpp index 1939d3730d..2ebd83fc93 100644 --- a/library/cpp/uri/uri_ut.cpp +++ b/library/cpp/uri/uri_ut.cpp @@ -4,7 +4,7 @@ #include <library/cpp/html/entity/htmlentity.h> #include <util/system/maxlen.h> - + namespace NUri { Y_UNIT_TEST_SUITE(URLTest) { static const char* urls[] = { @@ -66,7 +66,7 @@ namespace NUri { UNIT_ASSERT_VALUES_EQUAL(er, TState::ParsedOK); UNIT_ASSERT(base.IsValidAbs()); UNIT_ASSERT_VALUES_EQUAL(base.PrintS(), urls[0]); - + TString errbuf; TStringOutput out(errbuf); const long mflag = TFeature::FeaturesAll; @@ -75,12 +75,12 @@ namespace NUri { UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, urls[i]); rel.Merge(base); UNIT_ASSERT_VALUES_EQUAL_C(rel.PrintS(), urls[i + 1], urls[i]); - + // try the same thing differently er = rel.Parse(urls[i], mflag, urls[0]); UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, urls[i]); UNIT_ASSERT_VALUES_EQUAL_C(rel.PrintS(), urls[i + 1], urls[i]); - + // lastly... er = abs.Parse(urls[i + 1], mflag); UNIT_ASSERT_VALUES_EQUAL(er, TState::ParsedOK); @@ -120,7 +120,7 @@ namespace NUri { const char* const result; TUri::TLinkType ltype; }; - + static const Link4Norm link4Norm[] = { {"http://www.alltest.ru/all.php?a=aberporth", "http://www.alltest.ru/all.php?a=domestic jobs", "", TUri::LinkIsBad}, {"http://www.alltest.ru/all.php?a=aberporth", "http://www.alltest.ru/all.php?a=domestic%20jobs", "http://www.alltest.ru/all.php?a=domestic%20jobs", TUri::LinkIsLocal}, @@ -130,7 +130,7 @@ namespace NUri { Y_UNIT_TEST(test_httpURLNormalize) { TUri normalizedLink; - + for (int i = 0; link4Norm[i].link; i++) { TUri base; TState::EParsed er = base.Parse(link4Norm[i].base); @@ -141,14 +141,14 @@ namespace NUri { UNIT_ASSERT_VALUES_EQUAL_C(s, link4Norm[i].result, link4Norm[i].link); } } - + static const char* urlsWithMultipleSlash[] = { "http://a/http://b", "http://a/http://b", "http://a/https://b", "http://a/https://b", "http://a/b://c", "http://a/b:/c", "http://a/b//c", "http://a/b/c", nullptr, nullptr}; - + Y_UNIT_TEST(test_httpURLPathOperation) { char copyUrl[URL_MAXLEN]; for (int i = 0; urlsWithMultipleSlash[i]; i += 2) { @@ -164,7 +164,7 @@ namespace NUri { UNIT_ASSERT_VALUES_EQUAL_C(uri.PrintS(), normurl, url); } } - + static const char* hostsForCheckHost[] = { "simplehost.ru", "third_level.host.ru", |