diff options
author | denplusplus <denplusplus@yandex-team.ru> | 2022-02-10 16:47:34 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:34 +0300 |
commit | 57c20d143e8a438cd76b9fdc3ca2e8ee3ac1f32a (patch) | |
tree | cc63639f8e502db19a82c20e2861c6d1edbf9fea /library/cpp | |
parent | 464ba3814a83db4f2d5327393b0b6eaf0c86bfd7 (diff) | |
download | ydb-57c20d143e8a438cd76b9fdc3ca2e8ee3ac1f32a.tar.gz |
Restoring authorship annotation for <denplusplus@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp')
36 files changed, 1392 insertions, 1392 deletions
diff --git a/library/cpp/bit_io/bitinput.h b/library/cpp/bit_io/bitinput.h index 85711eb7f9..68dfce3499 100644 --- a/library/cpp/bit_io/bitinput.h +++ b/library/cpp/bit_io/bitinput.h @@ -139,7 +139,7 @@ namespace NBitIO { bool retCode = ReadWordsImpl<bits>(r64); result = r64; - + return retCode; } diff --git a/library/cpp/charset/codepage.h b/library/cpp/charset/codepage.h index 30a02a4610..e54097d2fb 100644 --- a/library/cpp/charset/codepage.h +++ b/library/cpp/charset/codepage.h @@ -166,13 +166,13 @@ inline const char* NameByCharset(ECharset e) { return ::NCodepagePrivate::TCodepagesMap::Instance().NameByCharset(e); } -inline const char* NameByCharsetSafe(ECharset e) { - if (CODES_UNKNOWN < e && e < CODES_MAX) +inline const char* NameByCharsetSafe(ECharset e) { + if (CODES_UNKNOWN < e && e < CODES_MAX) return ::NCodepagePrivate::TCodepagesMap::Instance().NameByCharset(e); - else + else ythrow yexception() << "unknown encoding: " << (int)e; -} - +} + inline const char* NameByCodePage(const CodePage* CP) { return CP->Names[0]; } diff --git a/library/cpp/charset/ya.make b/library/cpp/charset/ya.make index 7565566bf0..7dc1014329 100644 --- a/library/cpp/charset/ya.make +++ b/library/cpp/charset/ya.make @@ -15,8 +15,8 @@ SRCS( wide.cpp ) -PEERDIR( +PEERDIR( contrib/libs/libiconv -) - +) + END() diff --git a/library/cpp/containers/comptrie/chunked_helpers_trie.h b/library/cpp/containers/comptrie/chunked_helpers_trie.h index cfa35f5ba2..4095ca8b19 100644 --- a/library/cpp/containers/comptrie/chunked_helpers_trie.h +++ b/library/cpp/containers/comptrie/chunked_helpers_trie.h @@ -1,218 +1,218 @@ -#pragma once - +#pragma once + #include <library/cpp/on_disk/chunks/chunked_helpers.h> - -#include "comptrie.h" - -class TTrieSet { -private: - TCompactTrie<char> Trie; - -public: - TTrieSet(const TBlob& blob) - : Trie(blob) - { - } - - bool Has(const char* key) const { - return Trie.Find(key, strlen(key)); - } - - bool FindLongestPrefix(const char* key, size_t keylen, size_t* prefixLen) { - return Trie.FindLongestPrefix(key, keylen, prefixLen); - } -}; - + +#include "comptrie.h" + +class TTrieSet { +private: + TCompactTrie<char> Trie; + +public: + TTrieSet(const TBlob& blob) + : Trie(blob) + { + } + + bool Has(const char* key) const { + return Trie.Find(key, strlen(key)); + } + + bool FindLongestPrefix(const char* key, size_t keylen, size_t* prefixLen) { + return Trie.FindLongestPrefix(key, keylen, prefixLen); + } +}; + template <bool sorted = false> -class TTrieSetWriter { -private: - TCompactTrieBuilder<char> Builder; - -public: - TTrieSetWriter(bool isSorted = sorted) +class TTrieSetWriter { +private: + TCompactTrieBuilder<char> Builder; + +public: + TTrieSetWriter(bool isSorted = sorted) : Builder(isSorted ? CTBF_PREFIX_GROUPED : CTBF_NONE) - { - } - - void Add(const char* key, size_t keylen) { - Builder.Add(key, keylen, 0); + { + } + + void Add(const char* key, size_t keylen) { + Builder.Add(key, keylen, 0); assert(Has(((TString)key).substr(0, keylen).data())); - } - - void Add(const char* key) { - Add(key, strlen(key)); - } - - bool Has(const char* key) const { - ui64 dummy; - return Builder.Find(key, strlen(key), &dummy); - } - + } + + void Add(const char* key) { + Add(key, strlen(key)); + } + + bool Has(const char* key) const { + ui64 dummy; + return Builder.Find(key, strlen(key), &dummy); + } + void Save(IOutputStream& out) const { - Builder.Save(out); - } - - void Clear() { - Builder.Clear(); - } -}; - + Builder.Save(out); + } + + void Clear() { + Builder.Clear(); + } +}; + template <bool isWriter, bool sorted = false> -struct TTrieSetG; - +struct TTrieSetG; + template <bool sorted> -struct TTrieSetG<false, sorted> { - typedef TTrieSet T; -}; - +struct TTrieSetG<false, sorted> { + typedef TTrieSet T; +}; + template <bool sorted> -struct TTrieSetG<true, sorted> { - typedef TTrieSetWriter<sorted> T; -}; - +struct TTrieSetG<true, sorted> { + typedef TTrieSetWriter<sorted> T; +}; + template <typename T> -class TTrieMap { -private: - TCompactTrie<char> Trie; +class TTrieMap { +private: + TCompactTrie<char> Trie; static_assert(sizeof(T) <= sizeof(ui64), "expect sizeof(T) <= sizeof(ui64)"); - -public: - TTrieMap(const TBlob& blob) - : Trie(blob) - { - } - - bool Get(const char* key, T* value) const { - ui64 trieValue; - if (Trie.Find(key, strlen(key), &trieValue)) { + +public: + TTrieMap(const TBlob& blob) + : Trie(blob) + { + } + + bool Get(const char* key, T* value) const { + ui64 trieValue; + if (Trie.Find(key, strlen(key), &trieValue)) { *value = ReadUnaligned<T>(&trieValue); - return true; - } else { - return false; - } - } - - T Get(const char* key, T def = T()) const { - ui64 trieValue; - if (Trie.Find(key, strlen(key), &trieValue)) { + return true; + } else { + return false; + } + } + + T Get(const char* key, T def = T()) const { + ui64 trieValue; + if (Trie.Find(key, strlen(key), &trieValue)) { return ReadUnaligned<T>(&trieValue); - } else { - return def; - } - } + } else { + return def; + } + } const TCompactTrie<char>& GetTrie() const { return Trie; } -}; - +}; + template <typename T, bool sorted = false> -class TTrieMapWriter { -private: - typedef TCompactTrieBuilder<char> TBuilder; - TBuilder Builder; +class TTrieMapWriter { +private: + typedef TCompactTrieBuilder<char> TBuilder; + TBuilder Builder; static_assert(sizeof(T) <= sizeof(ui64), "expect sizeof(T) <= sizeof(ui64)"); -#ifndef NDEBUG - bool IsSorted; -#endif - -public: - TTrieMapWriter(bool isSorted = sorted) +#ifndef NDEBUG + bool IsSorted; +#endif + +public: + TTrieMapWriter(bool isSorted = sorted) : Builder(isSorted ? CTBF_PREFIX_GROUPED : CTBF_NONE) -#ifndef NDEBUG - , IsSorted(isSorted) -#endif - { - } - - void Add(const char* key, const T& value) { +#ifndef NDEBUG + , IsSorted(isSorted) +#endif + { + } + + void Add(const char* key, const T& value) { ui64 intValue = 0; memcpy(&intValue, &value, sizeof(T)); Builder.Add(key, strlen(key), intValue); -#ifndef NDEBUG +#ifndef NDEBUG /* - if (!IsSorted) { - T test; - assert(Get(key, &test) && value == test); - } + if (!IsSorted) { + T test; + assert(Get(key, &test) && value == test); + } */ -#endif - } - +#endif + } + void Add(const TString& s, const T& value) { ui64 intValue = 0; memcpy(&intValue, &value, sizeof(T)); Builder.Add(s.data(), s.size(), intValue); - } - - bool Get(const char* key, T* value) const { - ui64 trieValue; - if (Builder.Find(key, strlen(key), &trieValue)) { + } + + bool Get(const char* key, T* value) const { + ui64 trieValue; + if (Builder.Find(key, strlen(key), &trieValue)) { *value = ReadUnaligned<T>(&trieValue); - return true; - } else { - return false; - } - } - - T Get(const char* key, T def = (T)0) const { - ui64 trieValue; - if (Builder.Find(key, strlen(key), &trieValue)) { + return true; + } else { + return false; + } + } + + T Get(const char* key, T def = (T)0) const { + ui64 trieValue; + if (Builder.Find(key, strlen(key), &trieValue)) { return ReadUnaligned<T>(&trieValue); - } else { - return def; - } - } - + } else { + return def; + } + } + void Save(IOutputStream& out, bool minimize = false) const { - if (minimize) { + if (minimize) { CompactTrieMinimize<TBuilder>(out, Builder, false); - } else { - Builder.Save(out); - } - } - - void Clear() { - Builder.Clear(); - } -}; - + } else { + Builder.Save(out); + } + } + + void Clear() { + Builder.Clear(); + } +}; + template <typename T> -class TTrieSortedMapWriter { -private: +class TTrieSortedMapWriter { +private: typedef std::pair<TString, T> TValue; typedef TVector<TValue> TValues; - TValues Values; - -public: + TValues Values; + +public: TTrieSortedMapWriter() = default; - - void Add(const char* key, const T& value) { - Values.push_back(TValue(key, value)); - } - + + void Add(const char* key, const T& value) { + Values.push_back(TValue(key, value)); + } + void Save(IOutputStream& out) { Sort(Values.begin(), Values.end()); - TTrieMapWriter<T, true> writer; - for (typename TValues::const_iterator toValue = Values.begin(); toValue != Values.end(); ++toValue) + TTrieMapWriter<T, true> writer; + for (typename TValues::const_iterator toValue = Values.begin(); toValue != Values.end(); ++toValue) writer.Add(toValue->first.data(), toValue->second); - writer.Save(out); - } - - void Clear() { - Values.clear(); - } -}; - + writer.Save(out); + } + + void Clear() { + Values.clear(); + } +}; + template <typename X, bool isWriter, bool sorted = false> -struct TTrieMapG; - +struct TTrieMapG; + template <typename X, bool sorted> -struct TTrieMapG<X, false, sorted> { - typedef TTrieMap<X> T; -}; - +struct TTrieMapG<X, false, sorted> { + typedef TTrieMap<X> T; +}; + template <typename X, bool sorted> -struct TTrieMapG<X, true, sorted> { - typedef TTrieMapWriter<X, sorted> T; -}; +struct TTrieMapG<X, true, sorted> { + typedef TTrieMapWriter<X, sorted> T; +}; diff --git a/library/cpp/containers/comptrie/comptrie_impl.cpp b/library/cpp/containers/comptrie/comptrie_impl.cpp index a116ab6d1e..d16fc076f6 100644 --- a/library/cpp/containers/comptrie/comptrie_impl.cpp +++ b/library/cpp/containers/comptrie/comptrie_impl.cpp @@ -1,5 +1,5 @@ #include "comptrie_impl.h" - + #include <util/system/rusage.h> #include <util/stream/output.h> diff --git a/library/cpp/containers/comptrie/comptrie_trie.h b/library/cpp/containers/comptrie/comptrie_trie.h index 40ec1e52b3..633a299e8e 100644 --- a/library/cpp/containers/comptrie/comptrie_trie.h +++ b/library/cpp/containers/comptrie/comptrie_trie.h @@ -95,12 +95,12 @@ public: return Get(key.data(), key.size()); } TData GetDefault(const TKeyBuf& key, const TData& def) const { - TData value; + TData value; if (!Find(key.data(), key.size(), &value)) - return def; - else - return value; - } + return def; + else + return value; + } const TBlob& Data() const { return DataHolder; diff --git a/library/cpp/containers/comptrie/comptrie_ut.cpp b/library/cpp/containers/comptrie/comptrie_ut.cpp index 74bee09b5d..39a0ccb068 100644 --- a/library/cpp/containers/comptrie/comptrie_ut.cpp +++ b/library/cpp/containers/comptrie/comptrie_ut.cpp @@ -11,7 +11,7 @@ #include <util/generic/vector.h> #include <util/generic/ptr.h> #include <util/generic/ylimits.h> - + #include <util/folder/dirut.h> #include <util/random/random.h> @@ -31,8 +31,8 @@ class TCompactTrieTest: public TTestBase { -private: - UNIT_TEST_SUITE(TCompactTrieTest); +private: + UNIT_TEST_SUITE(TCompactTrieTest); UNIT_TEST(TestTrie8); UNIT_TEST(TestTrie16); UNIT_TEST(TestTrie32); @@ -113,16 +113,16 @@ private: UNIT_TEST(TestPatternSearcherSimple); UNIT_TEST(TestPatternSearcherRandom); - UNIT_TEST_SUITE_END(); - - static const char* SampleData[]; + UNIT_TEST_SUITE_END(); + + static const char* SampleData[]; template <class T> void CreateTrie(IOutputStream& out, bool minimize, bool useFastLayout); template <class T> - void CheckData(const char* src, size_t len); - + void CheckData(const char* src, size_t len); + template <class T> void CheckUpperBound(const char* src, size_t len); @@ -195,10 +195,10 @@ public: void TestMinimizedTrieIterator32(); void TestPhraseSearch(); - void TestAddGet(); - void TestEmpty(); + void TestAddGet(); + void TestEmpty(); void TestUninitializedNonEmpty(); - void TestRandom(); + void TestRandom(); void TestFindTails(); void TestPrefixGrouped(); void CrashTestPrefixGrouped(); @@ -261,10 +261,10 @@ public: TFastRng<ui64>& rng ); void TestPatternSearcherRandom(); -}; - -UNIT_TEST_SUITE_REGISTRATION(TCompactTrieTest); - +}; + +UNIT_TEST_SUITE_REGISTRATION(TCompactTrieTest); + const char* TCompactTrieTest::SampleData[] = { "", "a", "b", "c", "d", @@ -275,7 +275,7 @@ const char* TCompactTrieTest::SampleData[] = { "fbbaa", "c\x85\xA4\xBF" // Just something outside ASCII. }; - + template <class T> typename TCompactTrie<T>::TKey MakeWideKey(const char* str, size_t len) { typename TCompactTrie<T>::TKey buffer; @@ -299,12 +299,12 @@ typename TCompactTrie<T>::TKey MakeWideKey(const TStringBuf& buf) { template <class T> void TCompactTrieTest::CreateTrie(IOutputStream& out, bool minimize, bool useFastLayout) { TCompactTrieBuilder<T> builder; - + for (auto& i : SampleData) { size_t len = strlen(i); builder.Add(MakeWideKey<T>(i, len), len * 2); - } + } TBufferOutput tmp2; IOutputStream& currentOutput = useFastLayout ? tmp2 : out; @@ -318,8 +318,8 @@ void TCompactTrieTest::CreateTrie(IOutputStream& out, bool minimize, bool useFas if (useFastLayout) { CompactTrieMakeFastLayout<TCompactTriePacker<T>>(out, tmp2.Buffer().Data(), tmp2.Buffer().Size(), false); } -} - +} + // Iterates over all strings of length <= 4 made of letters a-g. static bool LexicographicStep(TString& s) { if (s.length() < 4) { @@ -360,14 +360,14 @@ void TCompactTrieTest::CheckUpperBound(const char* data, size_t datalen) { template <class T> void TCompactTrieTest::CheckData(const char* data, size_t datalen) { TCompactTrie<T> trie(data, datalen); - + UNIT_ASSERT_VALUES_EQUAL(Y_ARRAY_SIZE(SampleData), trie.Size()); for (auto& i : SampleData) { size_t len = strlen(i); ui64 value = 0; size_t prefixLen = 0; - + typename TCompactTrie<T>::TKey key = MakeWideKey<T>(i, len); UNIT_ASSERT(trie.Find(key, &value)); UNIT_ASSERT_EQUAL(len * 2, value); @@ -385,9 +385,9 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); UNIT_ASSERT_EQUAL(1, prefixLen); UNIT_ASSERT_EQUAL(2, value); - + badkey = i; - badkey += "x"; + badkey += "x"; key = MakeWideKey<T>(badkey); UNIT_ASSERT(!trie.Find(key)); value = 1234; @@ -398,7 +398,7 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { UNIT_ASSERT_EQUAL(len * 2, value); UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, nullptr)); UNIT_ASSERT_EQUAL(len, prefixLen); - } + } TString testkey("fbbaa"); typename TCompactTrie<T>::TKey key = MakeWideKey<T>(testkey); @@ -417,8 +417,8 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { value = 12345678; UNIT_ASSERT(!trie.Find(key, &value)); UNIT_ASSERT_EQUAL(12345678, value); //Failed Find() should not change value -} - +} + template <class T> void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { typedef typename TCompactTrie<T>::TKey TKey; @@ -487,19 +487,19 @@ void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { template <class T> void TCompactTrieTest::TestTrie(bool minimize, bool useFastLayout) { - TBufferOutput bufout; + TBufferOutput bufout; CreateTrie<T>(bufout, minimize, useFastLayout); CheckData<T>(bufout.Buffer().Data(), bufout.Buffer().Size()); CheckUpperBound<T>(bufout.Buffer().Data(), bufout.Buffer().Size()); -} - +} + template <class T> void TCompactTrieTest::TestTrieIterator(bool minimize) { - TBufferOutput bufout; + TBufferOutput bufout; CreateTrie<T>(bufout, minimize, false); CheckIterator<T>(bufout.Buffer().Data(), bufout.Buffer().Size()); -} - +} + void TCompactTrieTest::TestTrie8() { TestTrie<char>(false, false); } @@ -586,30 +586,30 @@ void TCompactTrieTest::TestPhraseSearch() { UNIT_ASSERT(matches.size() == 0); } -void TCompactTrieTest::TestAddGet() { +void TCompactTrieTest::TestAddGet() { TCompactTrieBuilder<char> builder; builder.Add("abcd", 4, 1); builder.Add("acde", 4, 2); - ui64 dummy; + ui64 dummy; UNIT_ASSERT(builder.Find("abcd", 4, &dummy)); - UNIT_ASSERT(1 == dummy); + UNIT_ASSERT(1 == dummy); UNIT_ASSERT(builder.Find("acde", 4, &dummy)); - UNIT_ASSERT(2 == dummy); + UNIT_ASSERT(2 == dummy); UNIT_ASSERT(!builder.Find("fgdgfacde", 9, &dummy)); UNIT_ASSERT(!builder.Find("ab", 2, &dummy)); -} - -void TCompactTrieTest::TestEmpty() { +} + +void TCompactTrieTest::TestEmpty() { TCompactTrieBuilder<char> builder; ui64 dummy = 12345; size_t prefixLen; UNIT_ASSERT(!builder.Find("abc", 3, &dummy)); - TBufferOutput bufout; - builder.Save(bufout); - + TBufferOutput bufout; + builder.Save(bufout); + TCompactTrie<char> trie(bufout.Buffer().Data(), bufout.Buffer().Size()); UNIT_ASSERT(!trie.Find("abc", 3, &dummy)); - UNIT_ASSERT(!trie.Find("", 0, &dummy)); + UNIT_ASSERT(!trie.Find("", 0, &dummy)); UNIT_ASSERT(!trie.FindLongestPrefix("abc", 3, &prefixLen, &dummy)); UNIT_ASSERT(!trie.FindLongestPrefix("", 0, &prefixLen, &dummy)); UNIT_ASSERT_EQUAL(12345, dummy); @@ -624,8 +624,8 @@ void TCompactTrieTest::TestEmpty() { trieNull.FindPhrases(" ", 1, matches); // just to be sure it doesn't crash UNIT_ASSERT(trieNull.Begin() == trieNull.End()); -} - +} + void TCompactTrieTest::TestUninitializedNonEmpty() { TBufferOutput bufout; CreateTrie<char>(bufout, false, false); @@ -653,11 +653,11 @@ static char RandChar() { static TString RandStr(const size_t max) { size_t len = RandomNumber<size_t>() % max; TString key; - for (size_t j = 0; j < len; ++j) + for (size_t j = 0; j < len; ++j) key += RandChar(); - return key; -} - + return key; +} + template <class T, bool minimize> void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { const TStringBuf EMPTY_KEY = TStringBuf("", 1); @@ -676,16 +676,16 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { UNIT_ASSERT_C(builder.Find(key.data(), key.size(), &dummy), "key = " << HexEncode(TString(key))); UNIT_ASSERT(dummy == val); } - } + } - TBufferStream stream; - size_t len = builder.Save(stream); + TBufferStream stream; + size_t len = builder.Save(stream); TCompactTrie<char, typename T::TData, T> trie(stream.Buffer().Data(), len); - + TBufferStream buftmp; if (minimize) { CompactTrieMinimize<T>(buftmp, stream.Buffer().Data(), len, false); - } + } TCompactTrie<char, typename T::TData, T> trieMin(buftmp.Buffer().Data(), buftmp.Buffer().Size()); TCompactTrieBuilder<char, typename T::TData, T> prefixGroupedBuilder(CTBF_PREFIX_GROUPED); @@ -718,7 +718,7 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { UNIT_ASSERT_VALUES_EQUAL(stream.Buffer().Size(), prefixGroupedBuffer.Buffer().Size()); UNIT_ASSERT(0 == memcmp(stream.Buffer().Data(), prefixGroupedBuffer.Buffer().Data(), stream.Buffer().Size())); -} +} void TCompactTrieTest::TestRandom() { TestRandom<TIntPacker<ui64>, true>(1000, 1000); diff --git a/library/cpp/containers/comptrie/make_fast_layout.cpp b/library/cpp/containers/comptrie/make_fast_layout.cpp index ade78d7899..ac34ea2c66 100644 --- a/library/cpp/containers/comptrie/make_fast_layout.cpp +++ b/library/cpp/containers/comptrie/make_fast_layout.cpp @@ -3,7 +3,7 @@ #include "writeable_node.h" #include "write_trie_backwards.h" #include "comptrie_impl.h" - + #include <util/generic/hash.h> #include <util/generic/utility.h> diff --git a/library/cpp/containers/comptrie/minimize.cpp b/library/cpp/containers/comptrie/minimize.cpp index 80d0b25217..c802b176b4 100644 --- a/library/cpp/containers/comptrie/minimize.cpp +++ b/library/cpp/containers/comptrie/minimize.cpp @@ -3,7 +3,7 @@ #include "writeable_node.h" #include "write_trie_backwards.h" #include "comptrie_impl.h" - + #include <util/generic/hash.h> #include <util/generic/algorithm.h> diff --git a/library/cpp/containers/comptrie/node.h b/library/cpp/containers/comptrie/node.h index d6f4317db0..f84132ebde 100644 --- a/library/cpp/containers/comptrie/node.h +++ b/library/cpp/containers/comptrie/node.h @@ -1,5 +1,5 @@ #pragma once - + #include <cstddef> namespace NCompactTrie { diff --git a/library/cpp/containers/comptrie/write_trie_backwards.cpp b/library/cpp/containers/comptrie/write_trie_backwards.cpp index fd8c28b0ed..7019d0644b 100644 --- a/library/cpp/containers/comptrie/write_trie_backwards.cpp +++ b/library/cpp/containers/comptrie/write_trie_backwards.cpp @@ -5,7 +5,7 @@ #include <util/generic/buffer.h> #include <util/generic/vector.h> - + namespace NCompactTrie { size_t WriteTrieBackwards(IOutputStream& os, TReverseNodeEnumerator& enumerator, bool verbose) { if (verbose) { diff --git a/library/cpp/containers/comptrie/writeable_node.cpp b/library/cpp/containers/comptrie/writeable_node.cpp index 404003dbbd..eb4a9455d4 100644 --- a/library/cpp/containers/comptrie/writeable_node.cpp +++ b/library/cpp/containers/comptrie/writeable_node.cpp @@ -1,7 +1,7 @@ #include "writeable_node.h" #include "node.h" #include "comptrie_impl.h" - + namespace NCompactTrie { TWriteableNode::TWriteableNode() : LeafPos(nullptr) diff --git a/library/cpp/containers/comptrie/ya.make b/library/cpp/containers/comptrie/ya.make index 81352da4b2..7e859df864 100644 --- a/library/cpp/containers/comptrie/ya.make +++ b/library/cpp/containers/comptrie/ya.make @@ -25,11 +25,11 @@ SRCS( writeable_node.cpp ) -PEERDIR( +PEERDIR( library/cpp/packers library/cpp/containers/compact_vector library/cpp/on_disk/chunks util/draft -) - +) + END() diff --git a/library/cpp/containers/intrusive_rb_tree/rb_tree.h b/library/cpp/containers/intrusive_rb_tree/rb_tree.h index 0259452a14..0a833ae223 100644 --- a/library/cpp/containers/intrusive_rb_tree/rb_tree.h +++ b/library/cpp/containers/intrusive_rb_tree/rb_tree.h @@ -16,7 +16,7 @@ struct TRbTreeNodeBase { TBasePtr Parent_; TBasePtr Left_; TBasePtr Right_; - size_t Children_; + size_t Children_; inline TRbTreeNodeBase() noexcept { ReInitNode(); @@ -27,7 +27,7 @@ struct TRbTreeNodeBase { Parent_ = nullptr; Left_ = nullptr; Right_ = nullptr; - Children_ = 1; + Children_ = 1; } static TBasePtr MinimumNode(TBasePtr x) { @@ -43,19 +43,19 @@ struct TRbTreeNodeBase { return x; } - - static TBasePtr ByIndex(TBasePtr x, size_t index) { + + static TBasePtr ByIndex(TBasePtr x, size_t index) { if (x->Left_ != nullptr) { - if (index < x->Left_->Children_) - return ByIndex(x->Left_, index); - index -= x->Left_->Children_; - } - if (0 == index) - return x; - if (!x->Right_) - ythrow yexception() << "index not found"; - return ByIndex(x->Right_, index - 1); - } + if (index < x->Left_->Children_) + return ByIndex(x->Left_, index); + index -= x->Left_->Children_; + } + if (0 == index) + return x; + if (!x->Right_) + ythrow yexception() << "index not found"; + return ByIndex(x->Right_, index - 1); + } }; struct TRbTreeBaseIterator; @@ -397,7 +397,7 @@ public: return y; } - + template <class T1> inline size_t LessCount(const T1& k) const { auto x = LowerBound(k); @@ -544,8 +544,8 @@ void TRbGlobal<TDummy>::RotateLeft(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) { x->Parent_->Right_ = y; y->Left_ = x; x->Parent_ = y; - y->Children_ = x->Children_; - x->Children_ = ((x->Left_) ? x->Left_->Children_ : 0) + ((x->Right_) ? x->Right_->Children_ : 0) + 1; + y->Children_ = x->Children_; + x->Children_ = ((x->Left_) ? x->Left_->Children_ : 0) + ((x->Right_) ? x->Right_->Children_ : 0) + 1; } template <class TDummy> @@ -564,8 +564,8 @@ void TRbGlobal<TDummy>::RotateRight(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) x->Parent_->Left_ = y; y->Right_ = x; x->Parent_ = y; - y->Children_ = x->Children_; - x->Children_ = ((x->Left_) ? x->Left_->Children_ : 0) + ((x->Right_) ? x->Right_->Children_ : 0) + 1; + y->Children_ = x->Children_; + x->Children_ = ((x->Left_) ? x->Left_->Children_ : 0) + ((x->Right_) ? x->Right_->Children_ : 0) + 1; } template <class TDummy> diff --git a/library/cpp/coroutine/engine/coroutine_ut.cpp b/library/cpp/coroutine/engine/coroutine_ut.cpp index 8b372496a2..65af04c97d 100644 --- a/library/cpp/coroutine/engine/coroutine_ut.cpp +++ b/library/cpp/coroutine/engine/coroutine_ut.cpp @@ -223,8 +223,8 @@ void TCoroTest::TestSimpleX2() { UNIT_ASSERT_EQUAL(i0, 100000); } -} - +} + struct TRunner { inline TRunner() : Runs(0) diff --git a/library/cpp/deprecated/kmp/kmp.cpp b/library/cpp/deprecated/kmp/kmp.cpp index d02074c94a..d9bb37fb5b 100644 --- a/library/cpp/deprecated/kmp/kmp.cpp +++ b/library/cpp/deprecated/kmp/kmp.cpp @@ -1,21 +1,21 @@ #include "kmp.h" #include <util/generic/yexception.h> - -TKMPMatcher::TKMPMatcher(const char* patternBegin, const char* patternEnd) - : Pattern(patternBegin, patternEnd) -{ - ComputePrefixFunction(); -} - + +TKMPMatcher::TKMPMatcher(const char* patternBegin, const char* patternEnd) + : Pattern(patternBegin, patternEnd) +{ + ComputePrefixFunction(); +} + TKMPMatcher::TKMPMatcher(const TString& pattern) - : Pattern(pattern) -{ - ComputePrefixFunction(); -} - -void TKMPMatcher::ComputePrefixFunction() { - ssize_t* pf; + : Pattern(pattern) +{ + ComputePrefixFunction(); +} + +void TKMPMatcher::ComputePrefixFunction() { + ssize_t* pf; ::ComputePrefixFunction(Pattern.data(), Pattern.data() + Pattern.size(), &pf); - PrefixFunction.Reset(pf); -} + PrefixFunction.Reset(pf); +} diff --git a/library/cpp/deprecated/kmp/kmp.h b/library/cpp/deprecated/kmp/kmp.h index a7f72eece6..71b554516d 100644 --- a/library/cpp/deprecated/kmp/kmp.h +++ b/library/cpp/deprecated/kmp/kmp.h @@ -1,108 +1,108 @@ #pragma once - + #include <util/generic/ptr.h> #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> - + template <typename T> -void ComputePrefixFunction(const T* begin, const T* end, ssize_t** result) { +void ComputePrefixFunction(const T* begin, const T* end, ssize_t** result) { Y_ENSURE(begin != end, TStringBuf("empty pattern")); - ssize_t len = end - begin; - TArrayHolder<ssize_t> resultHolder(new ssize_t[len + 1]); - ssize_t i = 0; - ssize_t j = -1; - resultHolder[0] = -1; - while (i < len) { - while ((j >= 0) && (begin[j] != begin[i])) - j = resultHolder[j]; - ++i; - ++j; + ssize_t len = end - begin; + TArrayHolder<ssize_t> resultHolder(new ssize_t[len + 1]); + ssize_t i = 0; + ssize_t j = -1; + resultHolder[0] = -1; + while (i < len) { + while ((j >= 0) && (begin[j] != begin[i])) + j = resultHolder[j]; + ++i; + ++j; Y_ASSERT(i >= 0); Y_ASSERT(j >= 0); Y_ASSERT(j < len); - if ((i < len) && (begin[i] == begin[j])) - resultHolder[i] = resultHolder[j]; - else - resultHolder[i] = j; - } - *result = resultHolder.Release(); -} - -class TKMPMatcher { -private: - TArrayHolder<ssize_t> PrefixFunction; + if ((i < len) && (begin[i] == begin[j])) + resultHolder[i] = resultHolder[j]; + else + resultHolder[i] = j; + } + *result = resultHolder.Release(); +} + +class TKMPMatcher { +private: + TArrayHolder<ssize_t> PrefixFunction; TString Pattern; - - void ComputePrefixFunction(); - -public: - TKMPMatcher(const char* patternBegin, const char* patternEnd); + + void ComputePrefixFunction(); + +public: + TKMPMatcher(const char* patternBegin, const char* patternEnd); TKMPMatcher(const TString& pattern); - - bool SubStr(const char* begin, const char* end, const char*& result) const { + + bool SubStr(const char* begin, const char* end, const char*& result) const { Y_ASSERT(begin <= end); ssize_t m = Pattern.size(); - ssize_t n = end - begin; - ssize_t i, j; - for (i = 0, j = 0; (i < n) && (j < m); ++i, ++j) { - while ((j >= 0) && (Pattern[j] != begin[i])) - j = PrefixFunction[j]; - } - if (j == m) { - result = begin + i - m; - return true; - } else { - return false; - } - } -}; - + ssize_t n = end - begin; + ssize_t i, j; + for (i = 0, j = 0; (i < n) && (j < m); ++i, ++j) { + while ((j >= 0) && (Pattern[j] != begin[i])) + j = PrefixFunction[j]; + } + if (j == m) { + result = begin + i - m; + return true; + } else { + return false; + } + } +}; + template <typename T> -class TKMPStreamMatcher { -public: - class ICallback { - public: - virtual void OnMatch(const T* begin, const T* end) = 0; +class TKMPStreamMatcher { +public: + class ICallback { + public: + virtual void OnMatch(const T* begin, const T* end) = 0; virtual ~ICallback() = default; - }; - -private: - ICallback* Callback; - TArrayHolder<ssize_t> PrefixFunction; + }; + +private: + ICallback* Callback; + TArrayHolder<ssize_t> PrefixFunction; using TTVector = TVector<T>; - TTVector Pattern; - ssize_t State; - TTVector Candidate; - -public: - TKMPStreamMatcher(const T* patternBegin, const T* patternEnd, ICallback* callback) - : Callback(callback) - , Pattern(patternBegin, patternEnd) - , State(0) + TTVector Pattern; + ssize_t State; + TTVector Candidate; + +public: + TKMPStreamMatcher(const T* patternBegin, const T* patternEnd, ICallback* callback) + : Callback(callback) + , Pattern(patternBegin, patternEnd) + , State(0) , Candidate(Pattern.size()) - { - ssize_t* pf; - ComputePrefixFunction(patternBegin, patternEnd, &pf); - PrefixFunction.Reset(pf); - } + { + ssize_t* pf; + ComputePrefixFunction(patternBegin, patternEnd, &pf); + PrefixFunction.Reset(pf); + } - void Push(const T& symbol) { - while ((State >= 0) && (Pattern[State] != symbol)) { + void Push(const T& symbol) { + while ((State >= 0) && (Pattern[State] != symbol)) { Y_ASSERT(State <= (ssize_t) Pattern.size()); - State = PrefixFunction[State]; + State = PrefixFunction[State]; Y_ASSERT(State <= (ssize_t) Pattern.size()); - } - if (State >= 0) - Candidate[State] = symbol; - ++State; + } + if (State >= 0) + Candidate[State] = symbol; + ++State; if (State == (ssize_t) Pattern.size()) { - Callback->OnMatch(Candidate.begin(), Candidate.end()); - State = 0; - } - } - - void Clear() { - State = 0; - } -}; + Callback->OnMatch(Candidate.begin(), Candidate.end()); + State = 0; + } + } + + void Clear() { + State = 0; + } +}; diff --git a/library/cpp/deprecated/kmp/kmp_ut.cpp b/library/cpp/deprecated/kmp/kmp_ut.cpp index c2eda83c57..98a73a91e2 100644 --- a/library/cpp/deprecated/kmp/kmp_ut.cpp +++ b/library/cpp/deprecated/kmp/kmp_ut.cpp @@ -1,80 +1,80 @@ #include "kmp.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/output.h> - + static TVector<int> FindAll(const TString& pattern, const TString& string) { TVector<int> result; - TKMPMatcher kmp(pattern); - const char* pResult; - const char* begin = string.begin(); - const char* end = string.end(); - while (kmp.SubStr(begin, end, pResult)) { + TKMPMatcher kmp(pattern); + const char* pResult; + const char* begin = string.begin(); + const char* end = string.end(); + while (kmp.SubStr(begin, end, pResult)) { result.push_back(int(pResult - string.data())); begin = pResult + pattern.size(); - } - return result; -} - + } + return result; +} + class TTestKMP: public TTestBase { - UNIT_TEST_SUITE(TTestKMP); + UNIT_TEST_SUITE(TTestKMP); UNIT_TEST(Test); UNIT_TEST(TestStream); - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE_END(); -public: - void Test() { +public: + void Test() { TVector<int> ans = {0, 2}; - UNIT_ASSERT_EQUAL(FindAll("a", "aba"), ans); + UNIT_ASSERT_EQUAL(FindAll("a", "aba"), ans); ans = {0}; - UNIT_ASSERT_EQUAL(FindAll("aba", "aba"), ans); + UNIT_ASSERT_EQUAL(FindAll("aba", "aba"), ans); ans.clear(); - UNIT_ASSERT_EQUAL(FindAll("abad", "aba"), ans); + UNIT_ASSERT_EQUAL(FindAll("abad", "aba"), ans); ans = {0, 2}; - UNIT_ASSERT_EQUAL(FindAll("ab", "abab"), ans); - } - + UNIT_ASSERT_EQUAL(FindAll("ab", "abab"), ans); + } + class TKMPSimpleCallback: public TKMPStreamMatcher<int>::ICallback { - private: - int* Begin; - int* End; - int Count; - - public: - TKMPSimpleCallback(int* begin, int* end) - : Begin(begin) - , End(end) - , Count(0) - { - } + private: + int* Begin; + int* End; + int Count; + public: + TKMPSimpleCallback(int* begin, int* end) + : Begin(begin) + , End(end) + , Count(0) + { + } + void OnMatch(const int* begin, const int* end) override { - UNIT_ASSERT_EQUAL(end - begin, End - Begin); - const int* p0 = Begin; - const int* p1 = begin; - while (p0 < End) { - UNIT_ASSERT_EQUAL(*p0, *p1); - ++p0; - ++p1; - } - ++Count; - } - - int GetCount() const { - return Count; - } - }; - - void TestStream() { - int pattern[] = {2, 3}; - int data[] = {1, 2, 3, 5, 2, 2, 3, 2, 4, 3, 2}; - TKMPSimpleCallback callback(pattern, pattern + 2); - TKMPStreamMatcher<int> matcher(pattern, pattern + 2, &callback); + UNIT_ASSERT_EQUAL(end - begin, End - Begin); + const int* p0 = Begin; + const int* p1 = begin; + while (p0 < End) { + UNIT_ASSERT_EQUAL(*p0, *p1); + ++p0; + ++p1; + } + ++Count; + } + + int GetCount() const { + return Count; + } + }; + + void TestStream() { + int pattern[] = {2, 3}; + int data[] = {1, 2, 3, 5, 2, 2, 3, 2, 4, 3, 2}; + TKMPSimpleCallback callback(pattern, pattern + 2); + TKMPStreamMatcher<int> matcher(pattern, pattern + 2, &callback); for (auto& i : data) matcher.Push(i); - UNIT_ASSERT_EQUAL(2, callback.GetCount()); - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TTestKMP); + UNIT_ASSERT_EQUAL(2, callback.GetCount()); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TTestKMP); diff --git a/library/cpp/deprecated/mapped_file/mapped_file.cpp b/library/cpp/deprecated/mapped_file/mapped_file.cpp index b0e4511299..6e72eef5c4 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.cpp +++ b/library/cpp/deprecated/mapped_file/mapped_file.cpp @@ -42,15 +42,15 @@ void TMappedFile::init(const TString& name, size_t length, TFileMap::EOpenMode o newFile.swap(*this); newFile.term(); } - + void TMappedFile::init(const TFile& file, TFileMap::EOpenMode om, const char* dbgName) { THolder<TFileMap> map(new TFileMap(file, om)); TMappedFile newFile(map.Get(), dbgName); Y_UNUSED(map.Release()); newFile.swap(*this); newFile.term(); -} - +} + void TMappedFile::init(const TString& name, TFileMap::EOpenMode om) { THolder<TFileMap> map(new TFileMap(name, om)); TMappedFile newFile(map.Get(), name.data()); @@ -61,4 +61,4 @@ void TMappedFile::init(const TString& name, TFileMap::EOpenMode om) { void TMappedFile::flush() { Map_->Flush(); -} +} diff --git a/library/cpp/deprecated/mapped_file/mapped_file.h b/library/cpp/deprecated/mapped_file/mapped_file.h index 45859ed65a..9edd84672a 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.h +++ b/library/cpp/deprecated/mapped_file/mapped_file.h @@ -17,7 +17,7 @@ class TMappedFile { private: TFileMap* Map_; - + private: TMappedFile(TFileMap* map, const char* dbgName); @@ -34,19 +34,19 @@ public: Map_ = nullptr; init(name, TFileMap::oRdOnly); } - + TMappedFile(const TFile& file, TFileMap::EOpenMode om = TFileMap::oRdOnly, const char* dbgName = "unknown"); void init(const TString& name); - + void init(const TString& name, TFileMap::EOpenMode om); void init(const TString& name, size_t length, TFileMap::EOpenMode om); - + void init(const TFile&, TFileMap::EOpenMode om = TFileMap::oRdOnly, const char* dbgName = "unknown"); - void flush(); - + void flush(); + void term() { if (Map_) { Map_->Unmap(); @@ -57,12 +57,12 @@ public: size_t getSize() const { return (Map_ ? Map_->MappedSize() : 0); - } + } void* getData(size_t pos = 0) const { Y_ASSERT(!Map_ || (pos <= getSize())); return (Map_ ? (void*)((unsigned char*)Map_->Ptr() + pos) : nullptr); - } + } void precharge(size_t pos = 0, size_t size = (size_t)-1) const; diff --git a/library/cpp/deprecated/split/split_iterator.cpp b/library/cpp/deprecated/split/split_iterator.cpp index 32262d25bd..88790475a6 100644 --- a/library/cpp/deprecated/split/split_iterator.cpp +++ b/library/cpp/deprecated/split/split_iterator.cpp @@ -1,112 +1,112 @@ #include "split_iterator.h" #include <util/system/yassert.h> - + #include <cctype> #include <cstring> #include <cstdlib> -/****************** TSplitDelimiters2 ******************/ - +/****************** TSplitDelimiters2 ******************/ + TSplitDelimiters::TSplitDelimiters(const char* s) { - memset(Delims, 0, sizeof(Delims)); - while (*s) + memset(Delims, 0, sizeof(Delims)); + while (*s) Delims[(ui8) * (s++)] = true; -} - -/****************** TSplitBase ******************/ +} + +/****************** TSplitBase ******************/ TSplitBase::TSplitBase(const char* str, size_t length) : Str(str) , Len(length) -{ -} - +{ +} + TSplitBase::TSplitBase(const TString& s) : Str(s.data()) , Len(s.size()) -{ -} - -/****************** TDelimitersSplit ******************/ - +{ +} + +/****************** TDelimitersSplit ******************/ + TDelimitersSplit::TDelimitersSplit(const char* str, size_t length, const TSplitDelimiters& delimiters) : TSplitBase(str, length) , Delimiters(delimiters) -{ -} - +{ +} + TDelimitersSplit::TDelimitersSplit(const TString& s, const TSplitDelimiters& delimiters) : TSplitBase(s) , Delimiters(delimiters) -{ -} - +{ +} + size_t TDelimitersSplit::Begin() const { - size_t pos = 0; + size_t pos = 0; while ((pos < Len) && Delimiters.IsDelimiter(Str[pos])) - ++pos; - return pos; -} - + ++pos; + return pos; +} + TSizeTRegion TDelimitersSplit::Next(size_t& pos) const { - size_t begin = pos; + size_t begin = pos; while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos])) - ++pos; - TSizeTRegion result(begin, pos); - + ++pos; + TSizeTRegion result(begin, pos); + while ((pos < Len) && Delimiters.IsDelimiter(Str[pos])) - ++pos; - - return result; -} + ++pos; + return result; +} + TDelimitersSplit::TIterator TDelimitersSplit::Iterator() const { - return TIterator(*this); -} - -/****************** TDelimitersStrictSplit ******************/ - + return TIterator(*this); +} + +/****************** TDelimitersStrictSplit ******************/ + TDelimitersStrictSplit::TDelimitersStrictSplit(const char* str, size_t length, const TSplitDelimiters& delimiters) : TSplitBase(str, length) , Delimiters(delimiters) -{ -} - +{ +} + TDelimitersStrictSplit::TDelimitersStrictSplit(const TString& s, const TSplitDelimiters& delimiters) : TSplitBase(s) , Delimiters(delimiters) -{ -} - +{ +} + TDelimitersStrictSplit::TIterator TDelimitersStrictSplit::Iterator() const { - return TIterator(*this); -} - + return TIterator(*this); +} + TSizeTRegion TDelimitersStrictSplit::Next(size_t& pos) const { - size_t begin = pos; + size_t begin = pos; while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos])) - ++pos; - TSizeTRegion result(begin, pos); - - if (pos < Len) - ++pos; - - return result; -} - + ++pos; + TSizeTRegion result(begin, pos); + + if (pos < Len) + ++pos; + + return result; +} + size_t TDelimitersStrictSplit::Begin() const { - return 0; -} - -/****************** TScreenedDelimitersSplit ******************/ - + return 0; +} + +/****************** TScreenedDelimitersSplit ******************/ + TScreenedDelimitersSplit::TScreenedDelimitersSplit(const TString& s, const TSplitDelimiters& delimiters, const TSplitDelimiters& screens) - : TSplitBase(s) - , Delimiters(delimiters) - , Screens(screens) -{ -} - + : TSplitBase(s) + , Delimiters(delimiters) + , Screens(screens) +{ +} + TScreenedDelimitersSplit::TScreenedDelimitersSplit(const char* str, size_t length, const TSplitDelimiters& delimiters, const TSplitDelimiters& screens) : TSplitBase(str, length) , Delimiters(delimiters) @@ -114,174 +114,174 @@ TScreenedDelimitersSplit::TScreenedDelimitersSplit(const char* str, size_t lengt { } -TScreenedDelimitersSplit::TIterator TScreenedDelimitersSplit::Iterator() const { - return TIterator(*this); -} - -TSizeTRegion TScreenedDelimitersSplit::Next(size_t& pos) const { - size_t begin = pos; - bool screened = false; - while (pos < Len) { - if (Screens.IsDelimiter(Str[pos])) - screened = !screened; - if (Delimiters.IsDelimiter(Str[pos]) && !screened) - break; - ++pos; - } - TSizeTRegion result(begin, pos); - - if (pos < Len) - ++pos; - - return result; -} - -size_t TScreenedDelimitersSplit::Begin() const { - return 0; -} - -/****************** TDelimitersSplitWithoutTags ******************/ - +TScreenedDelimitersSplit::TIterator TScreenedDelimitersSplit::Iterator() const { + return TIterator(*this); +} + +TSizeTRegion TScreenedDelimitersSplit::Next(size_t& pos) const { + size_t begin = pos; + bool screened = false; + while (pos < Len) { + if (Screens.IsDelimiter(Str[pos])) + screened = !screened; + if (Delimiters.IsDelimiter(Str[pos]) && !screened) + break; + ++pos; + } + TSizeTRegion result(begin, pos); + + if (pos < Len) + ++pos; + + return result; +} + +size_t TScreenedDelimitersSplit::Begin() const { + return 0; +} + +/****************** TDelimitersSplitWithoutTags ******************/ + TDelimitersSplitWithoutTags::TDelimitersSplitWithoutTags(const char* str, size_t length, const TSplitDelimiters& delimiters) : TSplitBase(str, length) , Delimiters(delimiters) -{ -} - +{ +} + TDelimitersSplitWithoutTags::TDelimitersSplitWithoutTags(const TString& s, const TSplitDelimiters& delimiters) : TSplitBase(s) , Delimiters(delimiters) -{ -} - +{ +} + size_t TDelimitersSplitWithoutTags::SkipTag(size_t pos) const { Y_ASSERT('<' == Str[pos]); - while ((pos < Len) && ('>' != Str[pos])) - ++pos; - return pos + 1; -} - + while ((pos < Len) && ('>' != Str[pos])) + ++pos; + return pos + 1; +} + size_t TDelimitersSplitWithoutTags::SkipDelimiters(size_t pos) const { while (true) { while ((pos < Len) && Delimiters.IsDelimiter(Str[pos]) && ('<' != Str[pos])) - ++pos; + ++pos; if (pos < Len) { - if ('<' != Str[pos]) - break; - else - pos = SkipTag(pos); + if ('<' != Str[pos]) + break; + else + pos = SkipTag(pos); } else - break; - } - return pos; -} - + break; + } + return pos; +} + size_t TDelimitersSplitWithoutTags::Begin() const { - size_t pos = 0; - pos = SkipDelimiters(pos); - return pos; -} - + size_t pos = 0; + pos = SkipDelimiters(pos); + return pos; +} + TSizeTRegion TDelimitersSplitWithoutTags::Next(size_t& pos) const { - size_t begin = pos; + size_t begin = pos; while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos]) && ('<' != Str[pos])) - ++pos; - TSizeTRegion result(begin, pos); - - pos = SkipDelimiters(pos); - - return result; -} - + ++pos; + TSizeTRegion result(begin, pos); + + pos = SkipDelimiters(pos); + + return result; +} + TDelimitersSplitWithoutTags::TIterator TDelimitersSplitWithoutTags::Iterator() const { - return TIterator(*this); -} - -/****************** TCharSplit ******************/ - + return TIterator(*this); +} + +/****************** TCharSplit ******************/ + TCharSplit::TCharSplit(const char* str, size_t length) : TSplitBase(str, length) -{ -} - +{ +} + TCharSplit::TCharSplit(const TString& s) : TSplitBase(s) -{ -} - +{ +} + TCharSplit::TIterator TCharSplit::Iterator() const { - return TIterator(*this); -} - + return TIterator(*this); +} + TSizeTRegion TCharSplit::Next(size_t& pos) const { - TSizeTRegion result(pos, pos + 1); - ++pos; - return result; -} - + TSizeTRegion result(pos, pos + 1); + ++pos; + return result; +} + size_t TCharSplit::Begin() const { - return 0; -} - -/****************** TCharSplitWithoutTags ******************/ - -TCharSplitWithoutTags::TCharSplitWithoutTags(const char* str, size_t length) - : TSplitBase(str, length) -{ -} - + return 0; +} + +/****************** TCharSplitWithoutTags ******************/ + +TCharSplitWithoutTags::TCharSplitWithoutTags(const char* str, size_t length) + : TSplitBase(str, length) +{ +} + TCharSplitWithoutTags::TCharSplitWithoutTags(const TString& s) - : TSplitBase(s) -{ -} - + : TSplitBase(s) +{ +} + size_t TCharSplitWithoutTags::SkipTag(size_t pos) const { Y_ASSERT('<' == Str[pos]); - while ((pos < Len) && ('>' != Str[pos])) - ++pos; - return pos + 1; -} - + while ((pos < Len) && ('>' != Str[pos])) + ++pos; + return pos + 1; +} + size_t TCharSplitWithoutTags::SkipDelimiters(size_t pos) const { while (true) { if (pos < Len) { - if ('<' != Str[pos]) - break; - else - pos = SkipTag(pos); + if ('<' != Str[pos]) + break; + else + pos = SkipTag(pos); } else - break; - } - return pos; -} - + break; + } + return pos; +} + size_t TCharSplitWithoutTags::Begin() const { - size_t pos = 0; - pos = SkipDelimiters(pos); - return pos; -} - + size_t pos = 0; + pos = SkipDelimiters(pos); + return pos; +} + TSizeTRegion TCharSplitWithoutTags::Next(size_t& pos) const { - size_t begin = pos++; - TSizeTRegion result(begin, pos); - - pos = SkipDelimiters(pos); - - return result; -} - + size_t begin = pos++; + TSizeTRegion result(begin, pos); + + pos = SkipDelimiters(pos); + + return result; +} + TCharSplitWithoutTags::TIterator TCharSplitWithoutTags::Iterator() const { - return TIterator(*this); -} - + return TIterator(*this); +} + TSubstringSplitDelimiter::TSubstringSplitDelimiter(const TString& s) - : Matcher(s) + : Matcher(s) , Len(s.size()) -{ -} - -/****************** TSubstringSplit ******************/ - +{ +} + +/****************** TSubstringSplit ******************/ + TSubstringSplit::TSubstringSplit(const char* str, size_t length, const TSubstringSplitDelimiter& delimiter) : TSplitBase(str, length) , Delimiter(delimiter) @@ -289,30 +289,30 @@ TSubstringSplit::TSubstringSplit(const char* str, size_t length, const TSubstrin } TSubstringSplit::TSubstringSplit(const TString& str, const TSubstringSplitDelimiter& delimiter) - : TSplitBase(str) - , Delimiter(delimiter) -{ -} - -TSubstringSplit::TIterator TSubstringSplit::Iterator() const { - return TIterator(*this); -} - -TSizeTRegion TSubstringSplit::Next(size_t& pos) const { - const char* begin = Str + pos; - const char* end = Str + Len; - const char* delim; - if (Delimiter.Matcher.SubStr(begin, end, delim)) { - TSizeTRegion result(pos, delim - begin + pos); - pos += delim - begin + Delimiter.Len; - return result; - } else { - TSizeTRegion result(pos, end - begin + pos); - pos += end - begin; - return result; - } -} - -size_t TSubstringSplit::Begin() const { - return 0; -} + : TSplitBase(str) + , Delimiter(delimiter) +{ +} + +TSubstringSplit::TIterator TSubstringSplit::Iterator() const { + return TIterator(*this); +} + +TSizeTRegion TSubstringSplit::Next(size_t& pos) const { + const char* begin = Str + pos; + const char* end = Str + Len; + const char* delim; + if (Delimiter.Matcher.SubStr(begin, end, delim)) { + TSizeTRegion result(pos, delim - begin + pos); + pos += delim - begin + Delimiter.Len; + return result; + } else { + TSizeTRegion result(pos, end - begin + pos); + pos += end - begin; + return result; + } +} + +size_t TSubstringSplit::Begin() const { + return 0; +} diff --git a/library/cpp/deprecated/split/split_iterator.h b/library/cpp/deprecated/split/split_iterator.h index 0eacc29228..f6f73dae11 100644 --- a/library/cpp/deprecated/split/split_iterator.h +++ b/library/cpp/deprecated/split/split_iterator.h @@ -1,5 +1,5 @@ #pragma once - + #include <library/cpp/deprecated/kmp/kmp.h> #include <util/string/cast.h> #include <util/string/util.h> @@ -11,7 +11,7 @@ #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> - + #include <cstdio> template <typename T> @@ -57,225 +57,225 @@ inline TUi32Region FromString(const TString& s) { } class TSplitDelimiters { -private: - bool Delims[256]; - -public: +private: + bool Delims[256]; + +public: explicit TSplitDelimiters(const char* s); Y_FORCE_INLINE bool IsDelimiter(ui8 ch) const { - return Delims[ch]; - } -}; - + return Delims[ch]; + } +}; + template <class Split> -class TSplitIterator; - +class TSplitIterator; + class TSplitBase { -protected: - const char* Str; - size_t Len; - -public: - TSplitBase(const char* str, size_t length); +protected: + const char* Str; + size_t Len; + +public: + TSplitBase(const char* str, size_t length); TSplitBase(const TString& s); - + Y_FORCE_INLINE const char* GetString() const { - return Str; - } - + return Str; + } + Y_FORCE_INLINE size_t GetLength() const { - return Len; - } + return Len; + } private: // we don't own Str, make sure that no one calls us with temporary object TSplitBase(TString&&) = delete; -}; - -#ifdef _MSC_VER +}; + +#ifdef _MSC_VER #pragma warning(push) #pragma warning(disable : 4512) -#endif - +#endif + class TDelimitersSplit: public TSplitBase { -private: - const TSplitDelimiters& Delimiters; - -public: +private: + const TSplitDelimiters& Delimiters; + +public: using TIterator = TSplitIterator<TDelimitersSplit>; - friend class TSplitIterator<TDelimitersSplit>; - - TDelimitersSplit(const char* str, size_t length, const TSplitDelimiters& delimiters); + friend class TSplitIterator<TDelimitersSplit>; + + TDelimitersSplit(const char* str, size_t length, const TSplitDelimiters& delimiters); TDelimitersSplit(const TString& s, const TSplitDelimiters& delimiters); - TIterator Iterator() const; - TSizeTRegion Next(size_t& pos) const; - size_t Begin() const; + TIterator Iterator() const; + TSizeTRegion Next(size_t& pos) const; + size_t Begin() const; private: // we don't own Delimiters, make sure that no one calls us with temporary object TDelimitersSplit(const char*, size_t, TSplitDelimiters&&) = delete; TDelimitersSplit(const TString&, TSplitDelimiters&&) = delete; TDelimitersSplit(TString&&, const TSplitDelimiters&) = delete; -}; - +}; + class TDelimitersStrictSplit: public TSplitBase { -private: - const TSplitDelimiters& Delimiters; - -public: +private: + const TSplitDelimiters& Delimiters; + +public: using TIterator = TSplitIterator<TDelimitersStrictSplit>; - friend class TSplitIterator<TDelimitersStrictSplit>; - - TDelimitersStrictSplit(const char* str, size_t length, const TSplitDelimiters& delimiters); + friend class TSplitIterator<TDelimitersStrictSplit>; + + TDelimitersStrictSplit(const char* str, size_t length, const TSplitDelimiters& delimiters); TDelimitersStrictSplit(const TString& s, const TSplitDelimiters& delimiters); - TIterator Iterator() const; - TSizeTRegion Next(size_t& pos) const; - size_t Begin() const; + TIterator Iterator() const; + TSizeTRegion Next(size_t& pos) const; + size_t Begin() const; private: // we don't own Delimiters, make sure that no one calls us with temporary object TDelimitersStrictSplit(const char*, size_t, TSplitDelimiters&&) = delete; TDelimitersStrictSplit(const TString&, TSplitDelimiters&&) = delete; TDelimitersStrictSplit(TString&&, const TSplitDelimiters&) = delete; -}; - +}; + class TScreenedDelimitersSplit: public TSplitBase { -private: - const TSplitDelimiters& Delimiters; - const TSplitDelimiters& Screens; - -public: +private: + const TSplitDelimiters& Delimiters; + const TSplitDelimiters& Screens; + +public: using TIterator = TSplitIterator<TScreenedDelimitersSplit>; - friend class TSplitIterator<TScreenedDelimitersSplit>; - + friend class TSplitIterator<TScreenedDelimitersSplit>; + TScreenedDelimitersSplit(const char*, size_t, const TSplitDelimiters& delimiters, const TSplitDelimiters& screens); TScreenedDelimitersSplit(const TString& s, const TSplitDelimiters& delimiters, const TSplitDelimiters& screens); - TIterator Iterator() const; - TSizeTRegion Next(size_t& pos) const; - size_t Begin() const; + TIterator Iterator() const; + TSizeTRegion Next(size_t& pos) const; + size_t Begin() const; private: // we don't own Delimiters and Screens, make sure that no one calls us with temporary object TScreenedDelimitersSplit(TString&&, const TSplitDelimiters&, const TSplitDelimiters&) = delete; TScreenedDelimitersSplit(const TString&, TSplitDelimiters&&, const TSplitDelimiters&) = delete; TScreenedDelimitersSplit(const TString&, const TSplitDelimiters&, TSplitDelimiters&&) = delete; -}; - +}; + class TDelimitersSplitWithoutTags: public TSplitBase { -private: - const TSplitDelimiters& Delimiters; - size_t SkipTag(size_t pos) const; - size_t SkipDelimiters(size_t pos) const; - -public: +private: + const TSplitDelimiters& Delimiters; + size_t SkipTag(size_t pos) const; + size_t SkipDelimiters(size_t pos) const; + +public: using TIterator = TSplitIterator<TDelimitersSplitWithoutTags>; - friend class TSplitIterator<TDelimitersSplitWithoutTags>; - - TDelimitersSplitWithoutTags(const char* str, size_t length, const TSplitDelimiters& delimiters); + friend class TSplitIterator<TDelimitersSplitWithoutTags>; + + TDelimitersSplitWithoutTags(const char* str, size_t length, const TSplitDelimiters& delimiters); TDelimitersSplitWithoutTags(const TString& s, const TSplitDelimiters& delimiters); - TIterator Iterator() const; - TSizeTRegion Next(size_t& pos) const; - size_t Begin() const; + TIterator Iterator() const; + TSizeTRegion Next(size_t& pos) const; + size_t Begin() const; private: // we don't own Delimiters, make sure that no one calls us with temporary object TDelimitersSplitWithoutTags(const char*, size_t, TSplitDelimiters&&) = delete; TDelimitersSplitWithoutTags(const TString&, TSplitDelimiters&&) = delete; TDelimitersSplitWithoutTags(TString&&, const TSplitDelimiters&) = delete; -}; - +}; + class TCharSplit: public TSplitBase { -public: +public: using TIterator = TSplitIterator<TCharSplit>; - friend class TSplitIterator<TCharSplit>; - - TCharSplit(const char* str, size_t length); + friend class TSplitIterator<TCharSplit>; + + TCharSplit(const char* str, size_t length); TCharSplit(const TString& s); - TIterator Iterator() const; - TSizeTRegion Next(size_t& pos) const; - size_t Begin() const; + TIterator Iterator() const; + TSizeTRegion Next(size_t& pos) const; + size_t Begin() const; private: // we don't own Str, make sure that no one calls us with temporary object TCharSplit(TString&&) = delete; -}; - -#ifdef _MSC_VER +}; + +#ifdef _MSC_VER #pragma warning(pop) -#endif - +#endif + class TCharSplitWithoutTags: public TSplitBase { -private: - size_t SkipTag(size_t pos) const; - size_t SkipDelimiters(size_t pos) const; - -public: +private: + size_t SkipTag(size_t pos) const; + size_t SkipDelimiters(size_t pos) const; + +public: using TIterator = TSplitIterator<TCharSplitWithoutTags>; - friend class TSplitIterator<TCharSplitWithoutTags>; - - TCharSplitWithoutTags(const char* str, size_t length); + friend class TSplitIterator<TCharSplitWithoutTags>; + + TCharSplitWithoutTags(const char* str, size_t length); TCharSplitWithoutTags(const TString& s); - TIterator Iterator() const; - TSizeTRegion Next(size_t& pos) const; - size_t Begin() const; + TIterator Iterator() const; + TSizeTRegion Next(size_t& pos) const; + size_t Begin() const; private: // we don't own Str, make sure that no one calls us with temporary object TCharSplitWithoutTags(TString&&) = delete; -}; - +}; + class TSubstringSplitDelimiter { -public: - TKMPMatcher Matcher; - size_t Len; - +public: + TKMPMatcher Matcher; + size_t Len; + TSubstringSplitDelimiter(const TString& s); -}; - +}; + class TSubstringSplit: public TSplitBase { -private: - const TSubstringSplitDelimiter& Delimiter; - -public: +private: + const TSubstringSplitDelimiter& Delimiter; + +public: using TIterator = TSplitIterator<TSubstringSplit>; - friend class TSplitIterator<TSubstringSplit>; - + friend class TSplitIterator<TSubstringSplit>; + TSubstringSplit(const char* str, size_t length, const TSubstringSplitDelimiter& delimiter); TSubstringSplit(const TString& str, const TSubstringSplitDelimiter& delimiter); - TIterator Iterator() const; - TSizeTRegion Next(size_t& pos) const; - size_t Begin() const; + TIterator Iterator() const; + TSizeTRegion Next(size_t& pos) const; + size_t Begin() const; private: // we don't own Delimiters, make sure that no one calls us with temporary object TSubstringSplit(TString&&, const TSubstringSplitDelimiter&) = delete; TSubstringSplit(const TString&, TSubstringSplitDelimiter&&) = delete; -}; - +}; + template <class TSplit> class TSplitIterator { -protected: - const TSplit& Split; - size_t Pos; +protected: + const TSplit& Split; + size_t Pos; TString* CurrentStroka; - -public: - TSplitIterator(const TSplit& split) - : Split(split) - , Pos(Split.Begin()) + +public: + TSplitIterator(const TSplit& split) + : Split(split) + , Pos(Split.Begin()) , CurrentStroka(nullptr) - { - } - - virtual ~TSplitIterator() { - delete CurrentStroka; - } - - inline TSizeTRegion Next() { + { + } + + virtual ~TSplitIterator() { + delete CurrentStroka; + } + + inline TSizeTRegion Next() { Y_ENSURE(!Eof(), TStringBuf("eof reached")); - return Split.Next(Pos); - } + return Split.Next(Pos); + } TStringBuf NextTok() { if (Eof()) @@ -285,33 +285,33 @@ public: } const TString& NextString() { - if (!CurrentStroka) + if (!CurrentStroka) CurrentStroka = new TString(); - TSizeTRegion region = Next(); - CurrentStroka->assign(Split.Str, region.Begin, region.Length() - 1); - return *CurrentStroka; - } - - inline bool Eof() const { - return Pos >= Split.Len; - } - + TSizeTRegion region = Next(); + CurrentStroka->assign(Split.Str, region.Begin, region.Length() - 1); + return *CurrentStroka; + } + + inline bool Eof() const { + return Pos >= Split.Len; + } + TString GetTail() const { return TString(Split.Str + Pos); - } - - void Skip(size_t count) { - for (size_t i = 0; i < count; ++i) - Next(); - } -}; - + } + + void Skip(size_t count) { + for (size_t i = 0; i < count; ++i) + Next(); + } +}; + using TSplitTokens = TVector<TString>; - + template <typename TSplit> void Split(const TSplit& split, TSplitTokens* words) { - words->clear(); - TSplitIterator<TSplit> it(split); - while (!it.Eof()) + words->clear(); + TSplitIterator<TSplit> it(split); + while (!it.Eof()) words->push_back(it.NextString()); -} +} diff --git a/library/cpp/deprecated/split/split_iterator_ut.cpp b/library/cpp/deprecated/split/split_iterator_ut.cpp index be5069c4be..42951b6ab4 100644 --- a/library/cpp/deprecated/split/split_iterator_ut.cpp +++ b/library/cpp/deprecated/split/split_iterator_ut.cpp @@ -1,93 +1,93 @@ #include "split_iterator.h" - + #include <library/cpp/testing/unittest/registar.h> - + class TSplitIteratorTest: public TTestBase { - UNIT_TEST_SUITE(TSplitIteratorTest); + UNIT_TEST_SUITE(TSplitIteratorTest); UNIT_TEST(TestDelimiters); UNIT_TEST(TestDelimitersSplit); UNIT_TEST(TestDelimitersStrictSplit); UNIT_TEST(TestTail); UNIT_TEST(TestScreenedDelimitersSplit); UNIT_TEST(TestSubstringDelimiter); - UNIT_TEST_SUITE_END(); - -public: - void TestDelimiters(); - void TestDelimitersSplit(); - void TestDelimitersStrictSplit(); - void TestTail(); - void TestScreenedDelimitersSplit(); - void TestSubstringDelimiter(); -}; - + UNIT_TEST_SUITE_END(); + +public: + void TestDelimiters(); + void TestDelimitersSplit(); + void TestDelimitersStrictSplit(); + void TestTail(); + void TestScreenedDelimitersSplit(); + void TestSubstringDelimiter(); +}; + void TSplitIteratorTest::TestDelimiters() { - TSplitDelimiters delims("@"); - for (int i = 0; i < 256; ++i) + TSplitDelimiters delims("@"); + for (int i = 0; i < 256; ++i) if ('@' != i) { - UNIT_ASSERT(!delims.IsDelimiter((ui8)i)); + UNIT_ASSERT(!delims.IsDelimiter((ui8)i)); } else { - UNIT_ASSERT(delims.IsDelimiter((ui8)i)); - } -} - + UNIT_ASSERT(delims.IsDelimiter((ui8)i)); + } +} + void TSplitIteratorTest::TestDelimitersSplit() { - { + { TString s = "1a3b45cd"; TSplitDelimiters delims("abcd"); TDelimitersSplit split(s, delims); - TSplitTokens tokens; - Split(split, &tokens); + TSplitTokens tokens; + Split(split, &tokens); TSplitTokens pattern = {"1", "3", "45"}; - UNIT_ASSERT(tokens == pattern); - } - { + UNIT_ASSERT(tokens == pattern); + } + { TString s = "aaaaaa"; TSplitDelimiters delims("abcd"); TDelimitersSplit split(s, delims); - TSplitTokens tokens; - Split(split, &tokens); + TSplitTokens tokens; + Split(split, &tokens); TSplitTokens pattern = {}; - UNIT_ASSERT(tokens == pattern); - } -} - + UNIT_ASSERT(tokens == pattern); + } +} + void TSplitIteratorTest::TestDelimitersStrictSplit() { - { + { TString s = "grp@2"; TSplitDelimiters delims("@"); TDelimitersStrictSplit split(s, delims); - TSplitTokens tokens; - Split(split, &tokens); + TSplitTokens tokens; + Split(split, &tokens); TSplitTokens pattern = {"grp", "2"}; - UNIT_ASSERT(tokens == pattern); - } - - { + UNIT_ASSERT(tokens == pattern); + } + + { TString s = "@grp@2@@"; TSplitDelimiters delims("@"); TDelimitersStrictSplit split(s, delims); - TSplitTokens tokens; - Split(split, &tokens); + TSplitTokens tokens; + Split(split, &tokens); TSplitTokens pattern = {"", "grp", "2", ""}; - UNIT_ASSERT(tokens == pattern); - } -} - + UNIT_ASSERT(tokens == pattern); + } +} + void TSplitIteratorTest::TestTail() { TString s = "grp@2@4"; TSplitDelimiters delims("@"); TDelimitersSplit split(s, delims); - TDelimitersSplit::TIterator it = split.Iterator(); - UNIT_ASSERT_EQUAL(it.GetTail(), "grp@2@4"); - it.Next(); - UNIT_ASSERT_EQUAL(it.GetTail(), "2@4"); - it.Next(); - UNIT_ASSERT_EQUAL(it.GetTail(), "4"); - it.Next(); - UNIT_ASSERT_EQUAL(it.GetTail(), ""); -} - + TDelimitersSplit::TIterator it = split.Iterator(); + UNIT_ASSERT_EQUAL(it.GetTail(), "grp@2@4"); + it.Next(); + UNIT_ASSERT_EQUAL(it.GetTail(), "2@4"); + it.Next(); + UNIT_ASSERT_EQUAL(it.GetTail(), "4"); + it.Next(); + UNIT_ASSERT_EQUAL(it.GetTail(), ""); +} + void TSplitIteratorTest::TestScreenedDelimitersSplit() { { const TString s = "77.88.58.91 - - [28/Aug/2008:00:08:07 +0400] \"GET /export/mordashka.tgz HTTP/1.1\" 304 - \"-\" \"libwww-perl/5.805\" \"news.yandex.ru,80\" \"-\" \"-\" 1219867687 \"0\" 3283 2"; @@ -135,18 +135,18 @@ void TSplitIteratorTest::TestScreenedDelimitersSplit() { UNIT_ASSERT_EQUAL(it.NextString(), "3283"); UNIT_ASSERT_EQUAL(it.NextString(), "2"); } -} - +} + void TSplitIteratorTest::TestSubstringDelimiter() { const TString s = "a@@bb@@cc@c.d@@r"; - static const TSubstringSplitDelimiter delimiter("@@"); - const TSubstringSplit splitter(s, delimiter); - TSubstringSplit::TIterator it = splitter.Iterator(); + static const TSubstringSplitDelimiter delimiter("@@"); + const TSubstringSplit splitter(s, delimiter); + TSubstringSplit::TIterator it = splitter.Iterator(); UNIT_ASSERT_EQUAL(it.NextString(), "a"); UNIT_ASSERT_EQUAL(it.NextString(), "bb"); UNIT_ASSERT_EQUAL(it.NextString(), "cc@c.d"); UNIT_ASSERT_EQUAL(it.NextString(), "r"); - UNIT_ASSERT(it.Eof()); -} - -UNIT_TEST_SUITE_REGISTRATION(TSplitIteratorTest); + UNIT_ASSERT(it.Eof()); +} + +UNIT_TEST_SUITE_REGISTRATION(TSplitIteratorTest); diff --git a/library/cpp/digest/old_crc/crc.h b/library/cpp/digest/old_crc/crc.h index 4a3ce6d05e..f734a087b5 100644 --- a/library/cpp/digest/old_crc/crc.h +++ b/library/cpp/digest/old_crc/crc.h @@ -21,8 +21,8 @@ inline ui16 crc16(const char* buf, size_t buflen, ui32 crcinit = CRC16INIT) { struct IdTR { Y_FORCE_INLINE static ui8 T(ui8 a) { - return a; - } + return a; + } }; // CCITT CRC-32 diff --git a/library/cpp/on_disk/chunks/chunked_helpers.cpp b/library/cpp/on_disk/chunks/chunked_helpers.cpp index b7adba2753..ad9f918751 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.cpp +++ b/library/cpp/on_disk/chunks/chunked_helpers.cpp @@ -1,21 +1,21 @@ -#include <util/ysaveload.h> - -#include "chunked_helpers.h" - -TBlob GetBlock(const TBlob& blob, size_t index) { - TChunkedDataReader reader(blob); +#include <util/ysaveload.h> + +#include "chunked_helpers.h" + +TBlob GetBlock(const TBlob& blob, size_t index) { + TChunkedDataReader reader(blob); if (index >= reader.GetBlocksCount()) ythrow yexception() << "index " << index << " is >= than block count " << reader.GetBlocksCount(); - size_t begin = (const char*)reader.GetBlock(index) - (const char*)blob.Data(); - return blob.SubBlob(begin, begin + reader.GetBlockLen(index)); -} + size_t begin = (const char*)reader.GetBlock(index) - (const char*)blob.Data(); + return blob.SubBlob(begin, begin + reader.GetBlockLen(index)); +} /*************************** TNamedChunkedDataReader ***************************/ static const char* NamedChunkedDataMagic = "NamedChunkedData"; -TNamedChunkedDataReader::TNamedChunkedDataReader(const TBlob& blob) - : TChunkedDataReader(blob) +TNamedChunkedDataReader::TNamedChunkedDataReader(const TBlob& blob) + : TChunkedDataReader(blob) { if (TChunkedDataReader::GetBlocksCount() < 1) throw yexception() << "Too few blocks"; diff --git a/library/cpp/on_disk/chunks/chunked_helpers.h b/library/cpp/on_disk/chunks/chunked_helpers.h index 5fa96afdca..fa7febf080 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.h +++ b/library/cpp/on_disk/chunks/chunked_helpers.h @@ -1,114 +1,114 @@ #pragma once - + #include <util/generic/vector.h> -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/generic/hash_set.h> -#include <util/generic/cast.h> +#include <util/generic/cast.h> #include <util/generic/ymath.h> #include <util/memory/blob.h> #include <util/stream/buffer.h> -#include <util/stream/mem.h> +#include <util/stream/mem.h> #include <util/system/unaligned_mem.h> #include <util/ysaveload.h> - -#include "reader.h" -#include "writer.h" - + +#include "reader.h" +#include "writer.h" + #include <cmath> #include <cstddef> template <typename T> -class TYVector { -private: - ui32 Size; - const T* Data; - -public: - TYVector(const TBlob& blob) +class TYVector { +private: + ui32 Size; + const T* Data; + +public: + TYVector(const TBlob& blob) : Size(IntegerCast<ui32>(ReadUnaligned<ui64>(blob.Data()))) , Data((const T*)((const char*)blob.Data() + sizeof(ui64))) - { - } - + { + } + void Get(size_t idx, T& t) const { assert(idx < (size_t)Size); t = ReadUnaligned<T>(Data + idx); } - const T& At(size_t idx) const { - assert(idx < (size_t)Size); - return Data[idx]; - } - - size_t GetSize() const { - return Size; - } - + const T& At(size_t idx) const { + assert(idx < (size_t)Size); + return Data[idx]; + } + + size_t GetSize() const { + return Size; + } + size_t RealSize() const { return sizeof(ui64) + Size * sizeof(T); } ~TYVector() = default; -}; - +}; + template <typename T> -class TYVectorWriter { -private: +class TYVectorWriter { +private: TVector<T> Vector; - -public: + +public: TYVectorWriter() = default; - - void PushBack(const T& value) { - Vector.push_back(value); - } - + + void PushBack(const T& value) { + Vector.push_back(value); + } + void Save(IOutputStream& out) const { - ui64 uSize = (ui64)Vector.size(); - out.Write(&uSize, sizeof(uSize)); + ui64 uSize = (ui64)Vector.size(); + out.Write(&uSize, sizeof(uSize)); out.Write(Vector.data(), Vector.size() * sizeof(T)); - } - - const T& At(size_t idx) const { - assert(idx < Size()); - return Vector[idx]; - } - - T& At(size_t idx) { - assert(idx < Size()); - return Vector[idx]; - } - - void Clear() { - Vector.clear(); - } - - size_t Size() const { - return Vector.size(); - } - - void Resize(size_t size) { - Vector.resize(size); - } - - void Resize(size_t size, const T& value) { - Vector.resize(size, value); - } -}; - + } + + const T& At(size_t idx) const { + assert(idx < Size()); + return Vector[idx]; + } + + T& At(size_t idx) { + assert(idx < Size()); + return Vector[idx]; + } + + void Clear() { + Vector.clear(); + } + + size_t Size() const { + return Vector.size(); + } + + void Resize(size_t size) { + Vector.resize(size); + } + + void Resize(size_t size, const T& value) { + Vector.resize(size, value); + } +}; + template <typename T, bool> -struct TYVectorG; - +struct TYVectorG; + template <typename X> -struct TYVectorG<X, false> { - typedef TYVector<X> T; -}; - +struct TYVectorG<X, false> { + typedef TYVector<X> T; +}; + template <typename X> -struct TYVectorG<X, true> { - typedef TYVectorWriter<X> T; -}; - +struct TYVectorG<X, true> { + typedef TYVectorWriter<X> T; +}; + template <typename T> struct TIsMemsetThisWithZeroesSupported { enum { @@ -124,16 +124,16 @@ struct TIsMemsetThisWithZeroesSupported { }; \ }; -class TPlainHashCommon { -protected: -#pragma pack(push, 8) +class TPlainHashCommon { +protected: +#pragma pack(push, 8) template <typename TKey, typename TValue> - class TPackedPair { - private: - typedef TPackedPair<TKey, TValue> TThis; - TKey Key; - TValue Value; - + class TPackedPair { + private: + typedef TPackedPair<TKey, TValue> TThis; + TKey Key; + TValue Value; + private: static_assert(TIsMemsetThisWithZeroesSupported<TKey>::Result, "expect TIsMemsetThisWithZeroesSupported<TKey>::Result"); static_assert(TIsMemsetThisWithZeroesSupported<TValue>::Result, "expect TIsMemsetThisWithZeroesSupported<TValue>::Result"); @@ -145,33 +145,33 @@ protected: Value = value; } - public: + public: TPackedPair(typename TTypeTraits<TKey>::TFuncParam key, typename TTypeTraits<TValue>::TFuncParam value) { Init(key, value); - } - + } + TPackedPair(const TThis& rhs) { Init(rhs.Key, rhs.Value); - } - - TPackedPair& operator=(const TThis& rhs) { - if (this != &rhs) { + } + + TPackedPair& operator=(const TThis& rhs) { + if (this != &rhs) { Init(rhs.Key, rhs.Value); - } - return *this; - } - + } + return *this; + } + TPackedPair() { Init(TKey(), TValue()); - } + } - typename TTypeTraits<TKey>::TFuncParam First() const { - return Key; - } - - typename TTypeTraits<TValue>::TFuncParam Second() const { - return Value; - } + typename TTypeTraits<TKey>::TFuncParam First() const { + return Key; + } + + typename TTypeTraits<TValue>::TFuncParam Second() const { + return Value; + } static TKey GetFirst(const void* self) { static constexpr size_t offset = offsetof(TThis, Key); @@ -182,29 +182,29 @@ protected: static constexpr size_t offset = offsetof(TThis, Value); return ReadUnaligned<TValue>(reinterpret_cast<const char*>(self) + offset); } - }; -#pragma pack(pop) - -protected: + }; +#pragma pack(pop) + +protected: static const ui16 VERSION_ID = 2; -#pragma pack(push, 8) - struct TInterval { - static const ui32 INVALID = (ui32)-1; - ui32 Offset; - ui32 Length; - - TInterval() - : Offset(INVALID) - , Length(INVALID) - { - } - - TInterval(ui32 offset, ui32 length) - : Offset(offset) - , Length(length) - { - } +#pragma pack(push, 8) + struct TInterval { + static const ui32 INVALID = (ui32)-1; + ui32 Offset; + ui32 Length; + + TInterval() + : Offset(INVALID) + , Length(INVALID) + { + } + + TInterval(ui32 offset, ui32 length) + : Offset(offset) + , Length(length) + { + } static inline ui32 GetOffset(const TInterval* self) { static constexpr size_t offset = offsetof(TInterval, Offset); @@ -215,244 +215,244 @@ protected: static constexpr size_t offset = offsetof(TInterval, Length); return ReadUnaligned<ui32>(reinterpret_cast<const char*>(self) + offset); } - }; -#pragma pack(pop) + }; +#pragma pack(pop) static_assert(8 == sizeof(TInterval), "expect 8 == sizeof(TInterval)"); template <typename TKey> - static ui32 KeyHash(typename TTypeTraits<TKey>::TFuncParam key, ui16 bits) { + static ui32 KeyHash(typename TTypeTraits<TKey>::TFuncParam key, ui16 bits) { Y_ASSERT(bits < 32); - const ui32 res = ui32(key) & ((ui32(1) << bits) - 1); + const ui32 res = ui32(key) & ((ui32(1) << bits) - 1); Y_ASSERT(res < (ui32(1) << bits)); - return res; - } -}; - + return res; + } +}; + template <typename TKey, typename TValue> -class TPlainHashWriter : TPlainHashCommon { -private: - typedef TPackedPair<TKey, TValue> TKeyValuePair; +class TPlainHashWriter : TPlainHashCommon { +private: + typedef TPackedPair<TKey, TValue> TKeyValuePair; typedef TVector<TKeyValuePair> TData; - TData Data; + TData Data; typedef TVector<TData> TData2; - - bool IsPlainEnought(ui16 bits) const { + + bool IsPlainEnought(ui16 bits) const { TVector<size_t> counts(1LL << bits, 0); for (size_t i = 0; i < Data.size(); ++i) { size_t& count = counts[KeyHash<TKey>(TKeyValuePair::GetFirst(&Data[i]), bits)]; - ++count; - if (count > 2) - return false; - } - return true; - } - -public: - void Add(const TKey& key, const TValue& value) { + ++count; + if (count > 2) + return false; + } + return true; + } + +public: + void Add(const TKey& key, const TValue& value) { Data.push_back(TKeyValuePair(key, value)); - } - + } + void Save(IOutputStream& out) const { Y_ASSERT(Data.size() < Max<ui32>()); WriteBin<ui16>(&out, VERSION_ID); - static const ui32 PAIR_SIZE = sizeof(TKeyValuePair); - WriteBin<ui32>(&out, PAIR_SIZE); + static const ui32 PAIR_SIZE = sizeof(TKeyValuePair); + WriteBin<ui32>(&out, PAIR_SIZE); - ui16 bits; - if (!Data.empty()) { + ui16 bits; + if (!Data.empty()) { bits = (ui16)(log((float)Data.size()) / log(2.f)); while ((bits < 22) && !IsPlainEnought(bits)) - ++bits; - } else { - bits = 0; - } - WriteBin<ui16>(&out, bits); + ++bits; + } else { + bits = 0; + } + WriteBin<ui16>(&out, bits); WriteBin<ui32>(&out, (ui32)Data.size()); - const ui32 nBuckets = ui32(1) << bits; - TData2 data2(nBuckets); + const ui32 nBuckets = ui32(1) << bits; + TData2 data2(nBuckets); for (size_t i = 0; i < Data.size(); ++i) data2[KeyHash<TKey>(TKeyValuePair::GetFirst(&Data[i]), bits)].push_back(Data[i]); typedef TVector<TInterval> TIntervals; - TIntervals intervals(nBuckets); - ui32 offset = 0; - for (ui32 i = 0; i < nBuckets; ++i) { - intervals[i].Offset = offset; + TIntervals intervals(nBuckets); + ui32 offset = 0; + for (ui32 i = 0; i < nBuckets; ++i) { + intervals[i].Offset = offset; intervals[i].Length = (ui32)data2[i].size(); offset += (ui32)data2[i].size(); - } -#ifndef NDEBUG - for (ui32 i = 0; i < nBuckets; ++i) { - for (size_t j = 0; j < data2[i].size(); ++j) - for (size_t k = j + 1; k < data2[i].size(); ++k) + } +#ifndef NDEBUG + for (ui32 i = 0; i < nBuckets; ++i) { + for (size_t j = 0; j < data2[i].size(); ++j) + for (size_t k = j + 1; k < data2[i].size(); ++k) if (TKeyValuePair::GetFirst(&data2[i][j]) == TKeyValuePair::GetFirst(&data2[i][k])) ythrow yexception() << "key clash"; - } -#endif + } +#endif out.Write(intervals.data(), intervals.size() * sizeof(intervals[0])); - for (ui32 i = 0; i < nBuckets; ++i) + for (ui32 i = 0; i < nBuckets; ++i) out.Write(data2[i].data(), data2[i].size() * sizeof(data2[i][0])); - } -}; - + } +}; + template <typename TKey, typename TValue> -class TPlainHash : TPlainHashCommon { -private: - typedef TPackedPair<TKey, TValue> TKeyValuePair; +class TPlainHash : TPlainHashCommon { +private: + typedef TPackedPair<TKey, TValue> TKeyValuePair; - const char* P; + const char* P; - ui16 GetBits() const { + ui16 GetBits() const { return ReadUnaligned<ui16>(P + 6); - } + } - ui32 GetSize() const { + ui32 GetSize() const { return ReadUnaligned<ui32>(P + 8); - } - - const TInterval* GetIntervals() const { - return (const TInterval*)(P + 12); - } + } - const TKeyValuePair* GetData() const { - return (const TKeyValuePair*)(GetIntervals() + (1ULL << GetBits())); - } + const TInterval* GetIntervals() const { + return (const TInterval*)(P + 12); + } + const TKeyValuePair* GetData() const { + return (const TKeyValuePair*)(GetIntervals() + (1ULL << GetBits())); + } + template <typename T> - void Init(const T* p) { + void Init(const T* p) { static_assert(sizeof(T) == 1, "expect sizeof(T) == 1"); - P = reinterpret_cast<const char*>(p); -#ifndef NDEBUG + P = reinterpret_cast<const char*>(p); +#ifndef NDEBUG ui16 version = ReadUnaligned<ui16>(p); if (version != VERSION_ID) - ythrow yexception() << "bad version: " << version; - static const ui32 PAIR_SIZE = sizeof(TKeyValuePair); + ythrow yexception() << "bad version: " << version; + static const ui32 PAIR_SIZE = sizeof(TKeyValuePair); const ui32 size = ReadUnaligned<ui32>(p + 2); - if (size != PAIR_SIZE) - ythrow yexception() << "bad size " << size << " instead of " << PAIR_SIZE; -#endif - } - -public: - typedef const TKeyValuePair* TConstIterator; - - TPlainHash(const char* p) { - Init(p); - } - - TPlainHash(const TBlob& blob) { - Init(blob.Begin()); - } - - bool Find(typename TTypeTraits<TKey>::TFuncParam key, TValue* res) const { - // Cerr << GetBits() << "\t" << (1 << GetBits()) << "\t" << GetSize() << Endl; - const ui32 hash = KeyHash<TKey>(key, GetBits()); + if (size != PAIR_SIZE) + ythrow yexception() << "bad size " << size << " instead of " << PAIR_SIZE; +#endif + } + +public: + typedef const TKeyValuePair* TConstIterator; + + TPlainHash(const char* p) { + Init(p); + } + + TPlainHash(const TBlob& blob) { + Init(blob.Begin()); + } + + bool Find(typename TTypeTraits<TKey>::TFuncParam key, TValue* res) const { + // Cerr << GetBits() << "\t" << (1 << GetBits()) << "\t" << GetSize() << Endl; + const ui32 hash = KeyHash<TKey>(key, GetBits()); const TInterval* intervalPtr = GetIntervals(); const TKeyValuePair* pair = GetData() + TInterval::GetOffset(intervalPtr + hash); const ui32 length = TInterval::GetLength(intervalPtr + hash); for (ui32 i = 0; i < length; ++i, ++pair) { if (TKeyValuePair::GetFirst(pair) == key) { *res = TKeyValuePair::GetSecond(pair); - return true; - } - } - return false; - } - - TValue Get(typename TTypeTraits<TKey>::TFuncParam key) const { - TValue res; - if (Find(key, &res)) - return res; - else - ythrow yexception() << "key not found"; - } - - TConstIterator Begin() const { - return GetData(); - } - - TConstIterator End() const { - return GetData() + GetSize(); - } - - const char* ByteEnd() const { - return (const char*)(GetData() + GetSize()); - } - - size_t ByteSize() const { + return true; + } + } + return false; + } + + TValue Get(typename TTypeTraits<TKey>::TFuncParam key) const { + TValue res; + if (Find(key, &res)) + return res; + else + ythrow yexception() << "key not found"; + } + + TConstIterator Begin() const { + return GetData(); + } + + TConstIterator End() const { + return GetData() + GetSize(); + } + + const char* ByteEnd() const { + return (const char*)(GetData() + GetSize()); + } + + size_t ByteSize() const { return 12 + sizeof(TInterval) * (size_t(1) << GetBits()) + sizeof(TKeyValuePair) * GetSize(); - } -}; - + } +}; + template <typename Key, typename Value, bool> -struct TPlainHashG; - +struct TPlainHashG; + template <typename Key, typename Value> -struct TPlainHashG<Key, Value, false> { - typedef TPlainHash<Key, Value> T; -}; - +struct TPlainHashG<Key, Value, false> { + typedef TPlainHash<Key, Value> T; +}; + template <typename Key, typename Value> -struct TPlainHashG<Key, Value, true> { - typedef TPlainHashWriter<Key, Value> T; -}; - +struct TPlainHashG<Key, Value, true> { + typedef TPlainHashWriter<Key, Value> T; +}; + template <typename T> -class TSingleValue { -private: - const T* Value; - -public: - TSingleValue(const TBlob& blob) { +class TSingleValue { +private: + const T* Value; + +public: + TSingleValue(const TBlob& blob) { Y_ASSERT(blob.Length() >= sizeof(T)); Y_ASSERT(blob.Length() <= sizeof(T) + 16); - Value = reinterpret_cast<const T*>(blob.Begin()); - } - - const T& Get() const { - return *Value; - } -}; - + Value = reinterpret_cast<const T*>(blob.Begin()); + } + + const T& Get() const { + return *Value; + } +}; + template <typename T> -class TSingleValueWriter { -private: - T Value; - -public: +class TSingleValueWriter { +private: + T Value; + +public: TSingleValueWriter() = default; - + TSingleValueWriter(const T& value) - : Value(value) - { - } - - void Set(const T& value) { - Value = value; - } - + : Value(value) + { + } + + void Set(const T& value) { + Value = value; + } + void Save(IOutputStream& out) const { - out.Write(&Value, sizeof(Value)); - } -}; - -TBlob GetBlock(const TBlob& data, size_t index); - + out.Write(&Value, sizeof(Value)); + } +}; + +TBlob GetBlock(const TBlob& data, size_t index); + template <class T> -void WriteBlock(TChunkedDataWriter& writer, const T& t) { - writer.NewBlock(); - t.Save(writer); -} - +void WriteBlock(TChunkedDataWriter& writer, const T& t) { + writer.NewBlock(); + t.Save(writer); +} + template <class T> -void WriteBlock(TChunkedDataWriter& writer, T& t) { - writer.NewBlock(); - t.Save(writer); -} - +void WriteBlock(TChunkedDataWriter& writer, T& t) { + writer.NewBlock(); + t.Save(writer); +} + // Extends TChunkedDataWriter, allowing user to name blocks with arbitrary strings. class TNamedChunkedDataWriter: public TChunkedDataWriter { public: diff --git a/library/cpp/on_disk/chunks/chunks_ut.cpp b/library/cpp/on_disk/chunks/chunks_ut.cpp index f727647f7f..8ef8f812a5 100644 --- a/library/cpp/on_disk/chunks/chunks_ut.cpp +++ b/library/cpp/on_disk/chunks/chunks_ut.cpp @@ -1,11 +1,11 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/file.h> #include <util/system/filemap.h> #include <util/system/tempfile.h> - -#include "chunked_helpers.h" - + +#include "chunked_helpers.h" + /// Data for TChunkedHelpersTest::TestGeneralVector struct TPodStruct { int x; @@ -82,13 +82,13 @@ public: void TestGeneralVector() { { /// ui32 const size_t N = 3; - TBufferStream stream; - { + TBufferStream stream; + { TGeneralVectorWriter<ui32> writer; for (size_t i = 0; i < N; ++i) writer.PushBack(i); - writer.Save(stream); - } + writer.Save(stream); + } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<ui32> reader(temp); @@ -138,7 +138,7 @@ public: TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<TItem> reader(temp); UNIT_ASSERT_EQUAL(reader.GetSize(), N); - + TItem value; reader.Get(0, value); UNIT_ASSERT(value.x == 1 && value.y == 2.0); @@ -154,14 +154,14 @@ public: TVector<int> data_holder(N); int* a = &(data_holder[0]); TBufferStream stream; - { + { TGeneralVectorWriter<int*> writer; for (size_t i = 0; i < N; ++i) { a[i] = i; writer.PushBack(a + i); } writer.Save(stream); - } + } { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<int*> reader(temp); @@ -174,7 +174,7 @@ public: } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(int*)); } - } + } { /// std::pair<int, int> typedef std::pair<int, int> TItem; const size_t N = 3; @@ -197,8 +197,8 @@ public: UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(TItem)); } } - } - + } + void TestStrings() { const TString FILENAME = "chunked_helpers_test.bin"; TTempFileHandle file(FILENAME.c_str()); diff --git a/library/cpp/on_disk/chunks/reader.cpp b/library/cpp/on_disk/chunks/reader.cpp index 6e28cbf367..af4fef0ecf 100644 --- a/library/cpp/on_disk/chunks/reader.cpp +++ b/library/cpp/on_disk/chunks/reader.cpp @@ -1,52 +1,52 @@ #include <util/generic/cast.h> -#include <util/memory/blob.h> +#include <util/memory/blob.h> #include <util/system/unaligned_mem.h> - -#include "reader.h" - -template <typename T> + +#include "reader.h" + +template <typename T> static inline void ReadAux(const char* data, T* aux, T count, TVector<const char*>* result) { - result->resize(count); - for (size_t i = 0; i < count; ++i) { + result->resize(count); + for (size_t i = 0; i < count; ++i) { (*result)[i] = data + ReadUnaligned<T>(aux + i); - } -} - + } +} + TChunkedDataReader::TChunkedDataReader(const TBlob& blob) { - const char* cdata = blob.AsCharPtr(); - const size_t size = blob.Size(); + const char* cdata = blob.AsCharPtr(); + const size_t size = blob.Size(); Y_ENSURE(size >= sizeof(ui32), "Empty file with chunks. "); ui32 last = ReadUnaligned<ui32>((ui32*)(cdata + size) - 1); - - if (last != 0) { // old version file - ui32* aux = (ui32*)(cdata + size); - ui32 count = last; - Size = size - (count + 1) * sizeof(ui32); - - aux -= (count + 1); - ReadAux<ui32>(cdata, aux, count, &Offsets); - return; - } - + + if (last != 0) { // old version file + ui32* aux = (ui32*)(cdata + size); + ui32 count = last; + Size = size - (count + 1) * sizeof(ui32); + + aux -= (count + 1); + ReadAux<ui32>(cdata, aux, count, &Offsets); + return; + } + Y_ENSURE(size >= 3 * sizeof(ui64), "Blob size must be >= 3 * sizeof(ui64). "); - ui64* aux = (ui64*)(cdata + size); + ui64* aux = (ui64*)(cdata + size); Version = ReadUnaligned<ui64>(aux - 2); Y_ENSURE(Version > 0, "Invalid chunked array version. "); - + ui64 count = ReadUnaligned<ui64>(aux - 3); - - aux -= (count + 3); - ReadAux<ui64>(cdata, aux, count, &Offsets); - - aux -= count; - Lengths.resize(count); - for (size_t i = 0; i < count; ++i) { + + aux -= (count + 3); + ReadAux<ui64>(cdata, aux, count, &Offsets); + + aux -= count; + Lengths.resize(count); + for (size_t i = 0; i < count; ++i) { Lengths[i] = IntegerCast<size_t>(ReadUnaligned<ui64>(aux + i)); - } -} - + } +} + TBlob TChunkedDataReader::GetBlob(size_t index) const { return TBlob::NoCopy(GetBlock(index), GetBlockLen(index)); } diff --git a/library/cpp/on_disk/chunks/reader.h b/library/cpp/on_disk/chunks/reader.h index c5fe783319..66b0155995 100644 --- a/library/cpp/on_disk/chunks/reader.h +++ b/library/cpp/on_disk/chunks/reader.h @@ -1,31 +1,31 @@ -#pragma once - +#pragma once + #include <util/generic/array_ref.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> - -class TBlob; - -class TChunkedDataReader { +#include <util/generic/vector.h> +#include <util/generic/yexception.h> + +class TBlob; + +class TChunkedDataReader { public: TChunkedDataReader(const TBlob& blob); - + inline const void* GetBlock(size_t index) const { CheckIndex(index); return Offsets[index]; } - + inline size_t GetBlockLen(size_t index) const { CheckIndex(index); - + if (Version == 0) { if (index + 1 < Offsets.size()) { return Offsets[index + 1] - Offsets[index]; - } - + } + return Size - (Offsets.back() - Offsets.front()); - } - + } + return Lengths[index]; } @@ -41,17 +41,17 @@ public: inline size_t GetBlocksCount() const { return Offsets.size(); } - + private: inline void CheckIndex(size_t index) const { if (index >= GetBlocksCount()) { ythrow yexception() << "requested block " << index << " of " << GetBlocksCount() << " blocks"; - } + } } - + private: ui64 Version = 0; TVector<const char*> Offsets; TVector<size_t> Lengths; size_t Size = 0; -}; +}; diff --git a/library/cpp/on_disk/chunks/writer.cpp b/library/cpp/on_disk/chunks/writer.cpp index 6dc7397f09..3c7747de0b 100644 --- a/library/cpp/on_disk/chunks/writer.cpp +++ b/library/cpp/on_disk/chunks/writer.cpp @@ -1,46 +1,46 @@ -#include <util/ysaveload.h> - -#include "writer.h" - +#include <util/ysaveload.h> + +#include "writer.h" + static inline void WriteAux(IOutputStream* out, const TVector<ui64>& data) { ::SavePodArray(out, data.data(), data.size()); -} - -/*************************** TBuffersWriter ***************************/ - +} + +/*************************** TBuffersWriter ***************************/ + TChunkedDataWriter::TChunkedDataWriter(IOutputStream& slave) - : Slave(slave) - , Offset(0) -{ -} - + : Slave(slave) + , Offset(0) +{ +} + TChunkedDataWriter::~TChunkedDataWriter() { -} - -void TChunkedDataWriter::NewBlock() { - if (Offsets.size()) { - Lengths.push_back(Offset - Offsets.back()); - } - - Pad(16); - Offsets.push_back(Offset); -} - -void TChunkedDataWriter::WriteFooter() { - Lengths.push_back(Offset - Offsets.back()); - WriteAux(this, Lengths); - WriteAux(this, Offsets); - WriteBinary<ui64>(Offsets.size()); - WriteBinary<ui64>(Version); - WriteBinary<ui64>(0); -} - -size_t TChunkedDataWriter::GetCurrentBlockOffset() const { +} + +void TChunkedDataWriter::NewBlock() { + if (Offsets.size()) { + Lengths.push_back(Offset - Offsets.back()); + } + + Pad(16); + Offsets.push_back(Offset); +} + +void TChunkedDataWriter::WriteFooter() { + Lengths.push_back(Offset - Offsets.back()); + WriteAux(this, Lengths); + WriteAux(this, Offsets); + WriteBinary<ui64>(Offsets.size()); + WriteBinary<ui64>(Version); + WriteBinary<ui64>(0); +} + +size_t TChunkedDataWriter::GetCurrentBlockOffset() const { Y_ASSERT(!Offsets.empty()); Y_ASSERT(Offset >= Offsets.back()); - return Offset - Offsets.back(); -} - -size_t TChunkedDataWriter::GetBlockCount() const { - return Offsets.size(); -} + return Offset - Offsets.back(); +} + +size_t TChunkedDataWriter::GetBlockCount() const { + return Offsets.size(); +} diff --git a/library/cpp/on_disk/chunks/writer.h b/library/cpp/on_disk/chunks/writer.h index ab14522bdd..ee0d7983c7 100644 --- a/library/cpp/on_disk/chunks/writer.h +++ b/library/cpp/on_disk/chunks/writer.h @@ -1,57 +1,57 @@ -#pragma once - -#include <util/generic/vector.h> -#include <util/stream/output.h> - -template <typename T> +#pragma once + +#include <util/generic/vector.h> +#include <util/stream/output.h> + +template <typename T> inline void WriteBin(IOutputStream* out, typename TTypeTraits<T>::TFuncParam t) { - out->Write(&t, sizeof(T)); -} - + out->Write(&t, sizeof(T)); +} + class TChunkedDataWriter: public IOutputStream { public: TChunkedDataWriter(IOutputStream& slave); ~TChunkedDataWriter() override; - + void NewBlock(); - + template <typename T> inline void WriteBinary(typename TTypeTraits<T>::TFuncParam t) { this->Write(&t, sizeof(T)); } - + void WriteFooter(); size_t GetCurrentBlockOffset() const; size_t GetBlockCount() const; - + protected: void DoWrite(const void* buf, size_t len) override { Slave.Write(buf, len); Offset += len; } - + private: static inline size_t PaddingSize(size_t size, size_t boundary) noexcept { const size_t boundaryViolation = size % boundary; - + return boundaryViolation == 0 ? 0 : boundary - boundaryViolation; } - + inline void Pad(size_t boundary) { const size_t newOffset = Offset + PaddingSize(Offset, boundary); - + while (Offset < newOffset) { Write('\0'); - } + } } - + private: static const ui64 Version = 1; - + IOutputStream& Slave; - + size_t Offset; TVector<ui64> Offsets; TVector<ui64> Lengths; -}; +}; diff --git a/library/cpp/on_disk/chunks/ya.make b/library/cpp/on_disk/chunks/ya.make index acb52df5b0..33d6f1e058 100644 --- a/library/cpp/on_disk/chunks/ya.make +++ b/library/cpp/on_disk/chunks/ya.make @@ -1,11 +1,11 @@ -LIBRARY() - +LIBRARY() + OWNER(g:util) -SRCS( - chunked_helpers.cpp - reader.cpp - writer.cpp -) - -END() +SRCS( + chunked_helpers.cpp + reader.cpp + writer.cpp +) + +END() diff --git a/library/cpp/packers/ut/packers_ut.cpp b/library/cpp/packers/ut/packers_ut.cpp index 18ce2150d1..3c048ababd 100644 --- a/library/cpp/packers/ut/packers_ut.cpp +++ b/library/cpp/packers/ut/packers_ut.cpp @@ -10,7 +10,7 @@ #include <util/generic/vector.h> #include <util/generic/ptr.h> #include <util/generic/ylimits.h> - + #include <util/folder/dirut.h> #include <util/random/random.h> @@ -23,11 +23,11 @@ #include <iterator> class TPackersTest: public TTestBase { -private: +private: UNIT_TEST_SUITE(TPackersTest); UNIT_TEST(TestPackers); - UNIT_TEST_SUITE_END(); - + UNIT_TEST_SUITE_END(); + template <class TData, class TPacker> void TestPacker(const TData& data); @@ -36,10 +36,10 @@ private: public: void TestPackers(); -}; - +}; + UNIT_TEST_SUITE_REGISTRATION(TPackersTest); - + template <class TData, class TPacker> void TPackersTest::TestPacker(const TData& data) { size_t len = TPacker().MeasureLeaf(data); diff --git a/library/cpp/string_utils/url/url.cpp b/library/cpp/string_utils/url/url.cpp index 85f4ac5d69..1a7a0f6ed7 100644 --- a/library/cpp/string_utils/url/url.cpp +++ b/library/cpp/string_utils/url/url.cpp @@ -51,7 +51,7 @@ namespace { if (!ignorehttps && urlSize.Has(8) && Compare1Case2(url, httpsPrefix, 8) == 0) return 8; return 0; - } + } template <typename T> inline T CutHttpPrefixImpl(const T& url, bool ignorehttps) { @@ -255,14 +255,14 @@ TStringBuf GetDomain(const TStringBuf host) noexcept { if (*c == '.') { if (wasPoint) { ++c; - break; + break; } - wasPoint = true; - } - } + wasPoint = true; + } + } return TStringBuf(c, host.end()); -} - +} + TStringBuf GetParentDomain(const TStringBuf host, size_t level) noexcept { size_t pos = host.size(); for (size_t i = 0; i < level; ++i) { @@ -275,8 +275,8 @@ TStringBuf GetParentDomain(const TStringBuf host, size_t level) noexcept { TStringBuf GetZone(const TStringBuf host) noexcept { return GetParentDomain(host, 1); -} - +} + TStringBuf CutWWWPrefix(const TStringBuf url) noexcept { if (url.size() >= 4 && url[3] == '.' && !strnicmp(url.data(), "www", 3)) return url.substr(4); @@ -342,8 +342,8 @@ static inline int x2c(unsigned char* x) { return X(x[0]) * 16 + X(x[1]); } -#undef X - +#undef X + static inline int Unescape(char* str) { char *to, *from; int dlen = 0; diff --git a/library/cpp/string_utils/url/url.h b/library/cpp/string_utils/url/url.h index 84137ccc57..a4d6b9a400 100644 --- a/library/cpp/string_utils/url/url.h +++ b/library/cpp/string_utils/url/url.h @@ -2,7 +2,7 @@ #include <util/generic/fwd.h> #include <util/generic/strbuf.h> - + namespace NUrl { /** diff --git a/library/cpp/string_utils/url/url_ut.cpp b/library/cpp/string_utils/url/url_ut.cpp index 1588013893..80085b31e3 100644 --- a/library/cpp/string_utils/url/url_ut.cpp +++ b/library/cpp/string_utils/url/url_ut.cpp @@ -26,7 +26,7 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { UNIT_ASSERT_VALUES_EQUAL("some_blender_url", GetHost("some_blender_url")); UNIT_ASSERT_VALUES_EQUAL("", GetHost("")); } - + Y_UNIT_TEST(TestGetPathAndQuery) { UNIT_ASSERT_VALUES_EQUAL("/", GetPathAndQuery("ru.wikipedia.org")); UNIT_ASSERT_VALUES_EQUAL("/", GetPathAndQuery("ru.wikipedia.org/")); @@ -40,14 +40,14 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { } Y_UNIT_TEST(TestGetDomain) { - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("www.ya.ru")); - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("ya.ru")); - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("a.b.ya.ru")); - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("ya.ru")); - UNIT_ASSERT_VALUES_EQUAL("ya", GetDomain("ya")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("www.ya.ru")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("ya.ru")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("a.b.ya.ru")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetDomain("ya.ru")); + UNIT_ASSERT_VALUES_EQUAL("ya", GetDomain("ya")); UNIT_ASSERT_VALUES_EQUAL("", GetDomain("")); - } - + } + Y_UNIT_TEST(TestGetParentDomain) { UNIT_ASSERT_VALUES_EQUAL("", GetParentDomain("www.ya.ru", 0)); UNIT_ASSERT_VALUES_EQUAL("ru", GetParentDomain("www.ya.ru", 1)); @@ -63,12 +63,12 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { } Y_UNIT_TEST(TestGetZone) { - UNIT_ASSERT_VALUES_EQUAL("ru", GetZone("www.ya.ru")); - UNIT_ASSERT_VALUES_EQUAL("com", GetZone("ya.com")); - UNIT_ASSERT_VALUES_EQUAL("RU", GetZone("RU")); - UNIT_ASSERT_VALUES_EQUAL("FHFBN", GetZone("ya.FHFBN")); - UNIT_ASSERT_VALUES_EQUAL("", GetZone("")); - } + UNIT_ASSERT_VALUES_EQUAL("ru", GetZone("www.ya.ru")); + UNIT_ASSERT_VALUES_EQUAL("com", GetZone("ya.com")); + UNIT_ASSERT_VALUES_EQUAL("RU", GetZone("RU")); + UNIT_ASSERT_VALUES_EQUAL("FHFBN", GetZone("ya.FHFBN")); + UNIT_ASSERT_VALUES_EQUAL("", GetZone("")); + } Y_UNIT_TEST(TestAddSchemePrefix) { UNIT_ASSERT_VALUES_EQUAL("http://yandex.ru", AddSchemePrefix("yandex.ru")); |