diff options
author | botay <botay@yandex-team.ru> | 2022-02-10 16:50:00 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:00 +0300 |
commit | c77325c5428270f74dd0d69adba028943a2b5ca8 (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | bdd495d3600d61815512f5dc4c1c02f65f047a16 (diff) | |
download | ydb-c77325c5428270f74dd0d69adba028943a2b5ca8.tar.gz |
Restoring authorship annotation for <botay@yandex-team.ru>. Commit 2 of 2.
-rw-r--r-- | library/cpp/on_disk/chunks/chunked_helpers.h | 282 | ||||
-rw-r--r-- | library/cpp/on_disk/chunks/chunks_ut.cpp | 438 | ||||
-rw-r--r-- | library/cpp/protobuf/util/pb_io.cpp | 4 | ||||
-rw-r--r-- | library/cpp/protobuf/util/pb_io.h | 2 | ||||
-rw-r--r-- | library/cpp/protobuf/util/simple_reflection.h | 18 | ||||
-rw-r--r-- | library/cpp/protobuf/util/simple_reflection_ut.cpp | 162 | ||||
-rw-r--r-- | library/cpp/protobuf/util/ut/sample_for_simple_reflection.proto | 14 | ||||
-rw-r--r-- | library/cpp/protobuf/util/ut/ya.make | 2 |
8 files changed, 461 insertions, 461 deletions
diff --git a/library/cpp/on_disk/chunks/chunked_helpers.h b/library/cpp/on_disk/chunks/chunked_helpers.h index 5f2215c5e1..5fa96afdca 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.h +++ b/library/cpp/on_disk/chunks/chunked_helpers.h @@ -6,10 +6,10 @@ #include <util/generic/cast.h> #include <util/generic/ymath.h> #include <util/memory/blob.h> -#include <util/stream/buffer.h> +#include <util/stream/buffer.h> #include <util/stream/mem.h> #include <util/system/unaligned_mem.h> -#include <util/ysaveload.h> +#include <util/ysaveload.h> #include "reader.h" #include "writer.h" @@ -30,11 +30,11 @@ public: { } - void Get(size_t idx, T& t) const { - assert(idx < (size_t)Size); + void Get(size_t idx, T& t) const { + assert(idx < (size_t)Size); t = ReadUnaligned<T>(Data + idx); - } - + } + const T& At(size_t idx) const { assert(idx < (size_t)Size); return Data[idx]; @@ -44,10 +44,10 @@ public: return Size; } - size_t RealSize() const { + size_t RealSize() const { return sizeof(ui64) + Size * sizeof(T); - } - + } + ~TYVector() = default; }; @@ -109,21 +109,21 @@ struct TYVectorG<X, true> { typedef TYVectorWriter<X> T; }; -template <typename T> -struct TIsMemsetThisWithZeroesSupported { - enum { - Result = TTypeTraits<T>::IsPod - }; -}; - +template <typename T> +struct TIsMemsetThisWithZeroesSupported { + enum { + Result = TTypeTraits<T>::IsPod + }; +}; + #define MEMSET_THIS_WITH_ZEROES_SUPPORTED(type) \ template <> \ struct TIsMemsetThisWithZeroesSupported<type> { \ enum { \ Result = true \ }; \ - }; - + }; + class TPlainHashCommon { protected: #pragma pack(push, 8) @@ -134,35 +134,35 @@ protected: TKey Key; TValue Value; - private: + private: static_assert(TIsMemsetThisWithZeroesSupported<TKey>::Result, "expect TIsMemsetThisWithZeroesSupported<TKey>::Result"); static_assert(TIsMemsetThisWithZeroesSupported<TValue>::Result, "expect TIsMemsetThisWithZeroesSupported<TValue>::Result"); - - /// to aviod uninitialized bytes - void Init(const TKey& key, const TValue& value) { - memset(static_cast<TThis*>(this), 0, sizeof(TThis)); - Key = key; - Value = value; - } - + + /// to aviod uninitialized bytes + void Init(const TKey& key, const TValue& value) { + memset(static_cast<TThis*>(this), 0, sizeof(TThis)); + Key = key; + Value = value; + } + public: - TPackedPair(typename TTypeTraits<TKey>::TFuncParam key, typename TTypeTraits<TValue>::TFuncParam value) { - Init(key, value); + TPackedPair(typename TTypeTraits<TKey>::TFuncParam key, typename TTypeTraits<TValue>::TFuncParam value) { + Init(key, value); } - TPackedPair(const TThis& rhs) { - Init(rhs.Key, rhs.Value); + TPackedPair(const TThis& rhs) { + Init(rhs.Key, rhs.Value); } TPackedPair& operator=(const TThis& rhs) { if (this != &rhs) { - Init(rhs.Key, rhs.Value); + Init(rhs.Key, rhs.Value); } return *this; } - TPackedPair() { - Init(TKey(), TValue()); + TPackedPair() { + Init(TKey(), TValue()); } typename TTypeTraits<TKey>::TFuncParam First() const { @@ -289,7 +289,7 @@ public: for (size_t j = 0; j < data2[i].size(); ++j) for (size_t k = j + 1; k < data2[i].size(); ++k) if (TKeyValuePair::GetFirst(&data2[i][j]) == TKeyValuePair::GetFirst(&data2[i][k])) - ythrow yexception() << "key clash"; + ythrow yexception() << "key clash"; } #endif out.Write(intervals.data(), intervals.size() * sizeof(intervals[0])); @@ -525,150 +525,150 @@ private: TVector<TString> Names; THashMap<TString, size_t> NameToIndex; }; - + template <class T> -struct TSaveLoadVectorNonPodElement { +struct TSaveLoadVectorNonPodElement { static inline void Save(IOutputStream* out, const T& t) { - TSerializer<T>::Save(out, t); - } - + TSerializer<T>::Save(out, t); + } + static inline void Load(IInputStream* in, T& t, size_t elementSize) { Y_ASSERT(elementSize > 0); - TSerializer<T>::Load(in, t); - } -}; - + TSerializer<T>::Load(in, t); + } +}; + template <class T, bool isPod> -class TVectorTakingIntoAccountThePodType { -private: - ui64 SizeofOffsets; - const ui64* Offsets; - const char* Data; - -public: - TVectorTakingIntoAccountThePodType(const TBlob& blob) { +class TVectorTakingIntoAccountThePodType { +private: + ui64 SizeofOffsets; + const ui64* Offsets; + const char* Data; + +public: + TVectorTakingIntoAccountThePodType(const TBlob& blob) { SizeofOffsets = ReadUnaligned<ui64>(blob.Begin()); Y_ASSERT(SizeofOffsets > 0); - Offsets = reinterpret_cast<const ui64*>(blob.Begin() + sizeof(ui64)); + Offsets = reinterpret_cast<const ui64*>(blob.Begin() + sizeof(ui64)); Data = reinterpret_cast<const char*>(blob.Begin() + sizeof(ui64) + SizeofOffsets * sizeof(ui64)); - } - - size_t GetSize() const { + } + + size_t GetSize() const { return (size_t)(SizeofOffsets - 1); - } - - size_t GetLength(ui64 index) const { - if (index + 1 >= SizeofOffsets) - ythrow yexception() << "bad offset"; + } + + size_t GetLength(ui64 index) const { + if (index + 1 >= SizeofOffsets) + ythrow yexception() << "bad offset"; return IntegerCast<size_t>(ReadUnaligned<ui64>(Offsets + index + 1) - ReadUnaligned<ui64>(Offsets + index)); - } - - void Get(ui64 index, T& t) const { - const size_t len = GetLength(index); + } + + void Get(ui64 index, T& t) const { + const size_t len = GetLength(index); TMemoryInput input(Data + ReadUnaligned<ui64>(Offsets + index), len); - TSaveLoadVectorNonPodElement<T>::Load(&input, t, len); - } - - T Get(ui64 index) const { - T ret; - Get(index, ret); - return ret; - } - - size_t RealSize() const { + TSaveLoadVectorNonPodElement<T>::Load(&input, t, len); + } + + T Get(ui64 index) const { + T ret; + Get(index, ret); + return ret; + } + + size_t RealSize() const { return sizeof(ui64) * (SizeofOffsets + 1) + ReadUnaligned<ui64>(Offsets + SizeofOffsets - 1); - } -}; - + } +}; + template <class T, bool isPod> -class TVectorTakingIntoAccountThePodTypeWriter : TNonCopyable { -private: +class TVectorTakingIntoAccountThePodTypeWriter : TNonCopyable { +private: typedef TVector<ui64> TOffsets; - TOffsets Offsets; - TBuffer Data; - TBufferOutput DataStream; - -public: - TVectorTakingIntoAccountThePodTypeWriter() - : DataStream(Data) - { - } - - void PushBack(const T& t) { + TOffsets Offsets; + TBuffer Data; + TBufferOutput DataStream; + +public: + TVectorTakingIntoAccountThePodTypeWriter() + : DataStream(Data) + { + } + + void PushBack(const T& t) { Offsets.push_back((ui64) Data.size()); - TSaveLoadVectorNonPodElement<T>::Save(&DataStream, t); - } - - size_t Size() const { - return Offsets.size(); - } - + TSaveLoadVectorNonPodElement<T>::Save(&DataStream, t); + } + + size_t Size() const { + return Offsets.size(); + } + void Save(IOutputStream& out) const { ui64 sizeofOffsets = Offsets.size() + 1; - out.Write(&sizeofOffsets, sizeof(sizeofOffsets)); + out.Write(&sizeofOffsets, sizeof(sizeofOffsets)); out.Write(Offsets.data(), Offsets.size() * sizeof(Offsets[0])); ui64 lastOffset = (ui64) Data.size(); - out.Write(&lastOffset, sizeof(lastOffset)); + out.Write(&lastOffset, sizeof(lastOffset)); out.Write(Data.data(), Data.size()); - } -}; - + } +}; + template <class T> class TVectorTakingIntoAccountThePodType<T, true>: public TYVector<T> { -public: - TVectorTakingIntoAccountThePodType(const TBlob& blob) - : TYVector<T>(blob) - { - } -}; - +public: + TVectorTakingIntoAccountThePodType(const TBlob& blob) + : TYVector<T>(blob) + { + } +}; + template <class T> class TVectorTakingIntoAccountThePodTypeWriter<T, true>: public TYVectorWriter<T> { -}; - +}; + template <typename T> class TGeneralVector: public TVectorTakingIntoAccountThePodType<T, TTypeTraits<T>::IsPod> { - typedef TVectorTakingIntoAccountThePodType<T, TTypeTraits<T>::IsPod> TBase; - -public: - TGeneralVector(const TBlob& blob) - : TBase(blob) - { - } -}; - + typedef TVectorTakingIntoAccountThePodType<T, TTypeTraits<T>::IsPod> TBase; + +public: + TGeneralVector(const TBlob& blob) + : TBase(blob) + { + } +}; + template <typename T> class TGeneralVectorWriter: public TVectorTakingIntoAccountThePodTypeWriter<T, TTypeTraits<T>::IsPod> { -}; - +}; + template <typename TItem, bool> -struct TGeneralVectorG; - +struct TGeneralVectorG; + template <typename TItem> -struct TGeneralVectorG<TItem, false> { - typedef TGeneralVector<TItem> T; -}; - +struct TGeneralVectorG<TItem, false> { + typedef TGeneralVector<TItem> T; +}; + template <typename TItem> -struct TGeneralVectorG<TItem, true> { - typedef TGeneralVectorWriter<TItem> T; -}; - +struct TGeneralVectorG<TItem, true> { + typedef TGeneralVectorWriter<TItem> T; +}; + template <> struct TSaveLoadVectorNonPodElement<TString> { static inline void Save(IOutputStream* out, const TString& s) { out->Write(s.data(), s.size() + 1); - } - + } + static inline void Load(TMemoryInput* in, TString& s, size_t elementSize) { Y_ASSERT(elementSize > 0 && in->Avail() >= elementSize); - s.assign(in->Buf(), elementSize - 1); /// excluding 0 at the end - } -}; - + s.assign(in->Buf(), elementSize - 1); /// excluding 0 at the end + } +}; + template <bool G> struct TStringsVectorG: public TGeneralVectorG<TString, G> { -}; - +}; + using TStringsVector = TGeneralVector<TString>; using TStringsVectorWriter = TGeneralVectorWriter<TString>; diff --git a/library/cpp/on_disk/chunks/chunks_ut.cpp b/library/cpp/on_disk/chunks/chunks_ut.cpp index 8410bda27b..f727647f7f 100644 --- a/library/cpp/on_disk/chunks/chunks_ut.cpp +++ b/library/cpp/on_disk/chunks/chunks_ut.cpp @@ -1,279 +1,279 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/file.h> -#include <util/system/filemap.h> -#include <util/system/tempfile.h> +#include <util/stream/file.h> +#include <util/system/filemap.h> +#include <util/system/tempfile.h> #include "chunked_helpers.h" -/// Data for TChunkedHelpersTest::TestGeneralVector -struct TPodStruct { - int x; - float y; - TPodStruct(int _x = 0, float _y = 0) - : x(_x) - , y(_y) - { - } -}; -/// And its serialization +/// Data for TChunkedHelpersTest::TestGeneralVector +struct TPodStruct { + int x; + float y; + TPodStruct(int _x = 0, float _y = 0) + : x(_x) + , y(_y) + { + } +}; +/// And its serialization template <> -struct TSaveLoadVectorNonPodElement<TPodStruct> { - typedef TPodStruct TItem; +struct TSaveLoadVectorNonPodElement<TPodStruct> { + typedef TPodStruct TItem; static inline void Save(IOutputStream* out, const TItem& item) { - TSerializer<int>::Save(out, item.x); - TSerializer<float>::Save(out, item.y); - } - + TSerializer<int>::Save(out, item.x); + TSerializer<float>::Save(out, item.y); + } + static inline void Load(IInputStream* in, TItem& item, size_t elementSize) { Y_ASSERT(elementSize == sizeof(TItem)); - TSerializer<int>::Load(in, item.x); - TSerializer<float>::Load(in, item.y); - } -}; - + TSerializer<int>::Load(in, item.x); + TSerializer<float>::Load(in, item.y); + } +}; + class TChunkedHelpersTest: public TTestBase { - UNIT_TEST_SUITE(TChunkedHelpersTest); + UNIT_TEST_SUITE(TChunkedHelpersTest); UNIT_TEST(TestHash) UNIT_TEST(TestGeneralVector) UNIT_TEST(TestStrings); UNIT_TEST(TestNamedChunkedData); - UNIT_TEST_SUITE_END(); - -public: - void TestHash() { - { - TBufferStream stream; - { - TPlainHashWriter<ui64, ui16> writer; - writer.Add(5, 7); - writer.Save(stream); - } - - { - TBlob temp = TBlob::FromStreamSingleThreaded(stream); - TPlainHash<ui64, ui16> reader(temp); + UNIT_TEST_SUITE_END(); + +public: + void TestHash() { + { + TBufferStream stream; + { + TPlainHashWriter<ui64, ui16> writer; + writer.Add(5, 7); + writer.Save(stream); + } + + { + TBlob temp = TBlob::FromStreamSingleThreaded(stream); + TPlainHash<ui64, ui16> reader(temp); ui16 value = 0; UNIT_ASSERT(reader.Find(5, &value)); UNIT_ASSERT_EQUAL(7, value); - UNIT_ASSERT(!reader.Find(6, &value)); - } - } - - { - TBufferStream stream; - int v = 1; - wchar16 k = 'a'; - { - TPlainHashWriter<wchar16, void*> writer; - writer.Add(k, &v); - writer.Save(stream); - } - { - TBlob temp = TBlob::FromStreamSingleThreaded(stream); - TPlainHash<wchar16, void*> reader(temp); + UNIT_ASSERT(!reader.Find(6, &value)); + } + } + + { + TBufferStream stream; + int v = 1; + wchar16 k = 'a'; + { + TPlainHashWriter<wchar16, void*> writer; + writer.Add(k, &v); + writer.Save(stream); + } + { + TBlob temp = TBlob::FromStreamSingleThreaded(stream); + TPlainHash<wchar16, void*> reader(temp); void* value = nullptr; - UNIT_ASSERT(reader.Find(k, &value)); - UNIT_ASSERT_EQUAL((int*)value, &v); - } - } - } - - void TestGeneralVector() { - { /// ui32 - const size_t N = 3; + UNIT_ASSERT(reader.Find(k, &value)); + UNIT_ASSERT_EQUAL((int*)value, &v); + } + } + } + + void TestGeneralVector() { + { /// ui32 + const size_t N = 3; TBufferStream stream; { - TGeneralVectorWriter<ui32> writer; - for (size_t i = 0; i < N; ++i) - writer.PushBack(i); + TGeneralVectorWriter<ui32> writer; + for (size_t i = 0; i < N; ++i) + writer.PushBack(i); writer.Save(stream); } - { + { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<ui32> reader(temp); - UNIT_ASSERT_EQUAL(reader.GetSize(), N); - for (size_t i = 0; i < N; ++i) { - ui32 value; - reader.Get(i, value); - UNIT_ASSERT_EQUAL(value, i); - UNIT_ASSERT_EQUAL(reader.At(i), i); - } + UNIT_ASSERT_EQUAL(reader.GetSize(), N); + for (size_t i = 0; i < N; ++i) { + ui32 value; + reader.Get(i, value); + UNIT_ASSERT_EQUAL(value, i); + UNIT_ASSERT_EQUAL(reader.At(i), i); + } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(ui32)); - } - } + } + } { /// TString - const size_t N = 4; - TBufferStream stream; - { + const size_t N = 4; + TBufferStream stream; + { TGeneralVectorWriter<TString> writer; - for (size_t i = 0; i < N; ++i) - writer.PushBack(ToString(i)); - writer.Save(stream); - } - { + for (size_t i = 0; i < N; ++i) + writer.PushBack(ToString(i)); + writer.Save(stream); + } + { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<TString> reader(temp); - UNIT_ASSERT_EQUAL(reader.GetSize(), N); - for (size_t i = 0; i < N; ++i) { + UNIT_ASSERT_EQUAL(reader.GetSize(), N); + for (size_t i = 0; i < N; ++i) { TString value; - reader.Get(i, value); - UNIT_ASSERT_EQUAL(value, ToString(i)); - UNIT_ASSERT_EQUAL(reader.Get(i), ToString(i)); - } + reader.Get(i, value); + UNIT_ASSERT_EQUAL(value, ToString(i)); + UNIT_ASSERT_EQUAL(reader.Get(i), ToString(i)); + } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * 2); - } - } - { /// some other struct - typedef TPodStruct TItem; - const size_t N = 2; - TBufferStream stream; - { - TGeneralVectorWriter<TItem> writer; - writer.PushBack(TItem(1, 2)); - writer.PushBack(TItem(3, 4)); - writer.Save(stream); - } - { + } + } + { /// some other struct + typedef TPodStruct TItem; + const size_t N = 2; + TBufferStream stream; + { + TGeneralVectorWriter<TItem> writer; + writer.PushBack(TItem(1, 2)); + writer.PushBack(TItem(3, 4)); + writer.Save(stream); + } + { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<TItem> reader(temp); - UNIT_ASSERT_EQUAL(reader.GetSize(), N); - - TItem value; - reader.Get(0, value); - UNIT_ASSERT(value.x == 1 && value.y == 2.0); - - reader.Get(1, value); - UNIT_ASSERT(value.x == 3 && value.y == 4.0); - + UNIT_ASSERT_EQUAL(reader.GetSize(), N); + + TItem value; + reader.Get(0, value); + UNIT_ASSERT(value.x == 1 && value.y == 2.0); + + reader.Get(1, value); + UNIT_ASSERT(value.x == 3 && value.y == 4.0); + UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * sizeof(TItem)); - } - } - { /// pointer - const size_t N = 3; + } + } + { /// pointer + const size_t N = 3; TVector<int> data_holder(N); - int* a = &(data_holder[0]); - TBufferStream stream; + int* a = &(data_holder[0]); + TBufferStream stream; { - TGeneralVectorWriter<int*> writer; - for (size_t i = 0; i < N; ++i) { - a[i] = i; - writer.PushBack(a + i); - } - writer.Save(stream); + TGeneralVectorWriter<int*> writer; + for (size_t i = 0; i < N; ++i) { + a[i] = i; + writer.PushBack(a + i); + } + writer.Save(stream); } - { + { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<int*> reader(temp); - UNIT_ASSERT_EQUAL(reader.GetSize(), N); - for (size_t i = 0; i < N; ++i) { - int* value; - reader.Get(i, value); - UNIT_ASSERT_EQUAL(value, a + i); - UNIT_ASSERT_EQUAL(reader.At(i), a + i); - } + UNIT_ASSERT_EQUAL(reader.GetSize(), N); + for (size_t i = 0; i < N; ++i) { + int* value; + reader.Get(i, value); + UNIT_ASSERT_EQUAL(value, a + i); + UNIT_ASSERT_EQUAL(reader.At(i), a + i); + } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(int*)); - } + } } { /// std::pair<int, int> typedef std::pair<int, int> TItem; - const size_t N = 3; - TBufferStream stream; - { - TGeneralVectorWriter<TItem> writer; - for (size_t i = 0; i < N; ++i) - writer.PushBack(TItem(i, i)); - writer.Save(stream); - } - { + const size_t N = 3; + TBufferStream stream; + { + TGeneralVectorWriter<TItem> writer; + for (size_t i = 0; i < N; ++i) + writer.PushBack(TItem(i, i)); + writer.Save(stream); + } + { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TGeneralVector<TItem> reader(temp); - UNIT_ASSERT_EQUAL(reader.GetSize(), N); - for (size_t i = 0; i < N; ++i) { - TItem value; - reader.Get(i, value); - UNIT_ASSERT_EQUAL(value, TItem(i, i)); - } + UNIT_ASSERT_EQUAL(reader.GetSize(), N); + for (size_t i = 0; i < N; ++i) { + TItem value; + reader.Get(i, value); + UNIT_ASSERT_EQUAL(value, TItem(i, i)); + } UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(TItem)); - } - } + } + } } - void TestStrings() { + void TestStrings() { const TString FILENAME = "chunked_helpers_test.bin"; - TTempFileHandle file(FILENAME.c_str()); + TTempFileHandle file(FILENAME.c_str()); - { + { TFixedBufferFileOutput fOut(FILENAME); - TStringsVectorWriter stringsWriter; - stringsWriter.PushBack(""); - stringsWriter.PushBack("test"); - TChunkedDataWriter writer(fOut); - WriteBlock(writer, stringsWriter); - writer.WriteFooter(); - } - - { + TStringsVectorWriter stringsWriter; + stringsWriter.PushBack(""); + stringsWriter.PushBack("test"); + TChunkedDataWriter writer(fOut); + WriteBlock(writer, stringsWriter); + writer.WriteFooter(); + } + + { TBlob fIn = TBlob::FromFileSingleThreaded(FILENAME); - TStringsVector vct(GetBlock(fIn, 0)); - UNIT_ASSERT_EQUAL(vct.Get(0), ""); - UNIT_ASSERT_EQUAL(vct.Get(1), "test"); - - bool wasException = false; - try { - vct.Get(2); - } catch (...) { - wasException = true; - } - UNIT_ASSERT(wasException); - } - } - - void TestNamedChunkedData() { + TStringsVector vct(GetBlock(fIn, 0)); + UNIT_ASSERT_EQUAL(vct.Get(0), ""); + UNIT_ASSERT_EQUAL(vct.Get(1), "test"); + + bool wasException = false; + try { + vct.Get(2); + } catch (...) { + wasException = true; + } + UNIT_ASSERT(wasException); + } + } + + void TestNamedChunkedData() { const TString filename = MakeTempName(nullptr, "named_chunked_data_test"); TTempFile file(filename); - - { + + { TFixedBufferFileOutput fOut(filename); - TNamedChunkedDataWriter writer(fOut); - - writer.NewBlock("alpha"); - writer.Write("123456"); - - writer.NewBlock(); - writer.Write("anonymous"); - - writer.NewBlock("omega"); - writer.Write("12345678901234567"); - - writer.WriteFooter(); - } - - { + TNamedChunkedDataWriter writer(fOut); + + writer.NewBlock("alpha"); + writer.Write("123456"); + + writer.NewBlock(); + writer.Write("anonymous"); + + writer.NewBlock("omega"); + writer.Write("12345678901234567"); + + writer.WriteFooter(); + } + + { TBlob mf = TBlob::FromFileSingleThreaded(filename); - TNamedChunkedDataReader reader(mf); - - UNIT_ASSERT(reader.GetBlocksCount() == 3); - - UNIT_ASSERT(reader.HasBlock("alpha")); - UNIT_ASSERT(reader.HasBlock("omega")); - - UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(0), "alpha"); - UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(1), ""); - UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(2), "omega"); - + TNamedChunkedDataReader reader(mf); + + UNIT_ASSERT(reader.GetBlocksCount() == 3); + + UNIT_ASSERT(reader.HasBlock("alpha")); + UNIT_ASSERT(reader.HasBlock("omega")); + + UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(0), "alpha"); + UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(1), ""); + UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(2), "omega"); + UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("alpha"), 6); // padding not included - UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("omega"), 17); - - UNIT_ASSERT(memcmp(reader.GetBlockByName("alpha"), "123456", 6) == 0); - UNIT_ASSERT(memcmp(reader.GetBlock(1), "anonymous", 9) == 0); - UNIT_ASSERT(memcmp(reader.GetBlockByName("omega"), "12345678901234567", 17) == 0); - } - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TChunkedHelpersTest); - + UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("omega"), 17); + + UNIT_ASSERT(memcmp(reader.GetBlockByName("alpha"), "123456", 6) == 0); + UNIT_ASSERT(memcmp(reader.GetBlock(1), "anonymous", 9) == 0); + UNIT_ASSERT(memcmp(reader.GetBlockByName("omega"), "12345678901234567", 17) == 0); + } + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TChunkedHelpersTest); + class TChunkedDataTest: public TTestBase { private: UNIT_TEST_SUITE(TChunkedDataTest); diff --git a/library/cpp/protobuf/util/pb_io.cpp b/library/cpp/protobuf/util/pb_io.cpp index 564f3ce2b5..6270ee0624 100644 --- a/library/cpp/protobuf/util/pb_io.cpp +++ b/library/cpp/protobuf/util/pb_io.cpp @@ -79,8 +79,8 @@ namespace NProtoBuf { bool MergeFromString(NProtoBuf::Message& m, const TStringBuf serializedProtoMessage) { return MergePartialFromString(m, serializedProtoMessage) && m.IsInitialized(); } -} - +} + int operator&(NProtoBuf::Message& m, IBinSaver& f) { TStringStream ss; if (f.IsReading()) { diff --git a/library/cpp/protobuf/util/pb_io.h b/library/cpp/protobuf/util/pb_io.h index eb9ca603e3..493c84cb5f 100644 --- a/library/cpp/protobuf/util/pb_io.h +++ b/library/cpp/protobuf/util/pb_io.h @@ -35,7 +35,7 @@ namespace NProtoBuf { TString SerializeToBase64String(const Message& m); void SerializeToBase64String(const Message& m, TString& dataBase64); bool TrySerializeToBase64String(const Message& m, TString& dataBase64); - + const TString ShortUtf8DebugString(const Message& message); bool MergePartialFromString(NProtoBuf::Message& m, const TStringBuf serializedProtoMessage); diff --git a/library/cpp/protobuf/util/simple_reflection.h b/library/cpp/protobuf/util/simple_reflection.h index fe470c53a8..61e877a787 100644 --- a/library/cpp/protobuf/util/simple_reflection.h +++ b/library/cpp/protobuf/util/simple_reflection.h @@ -103,7 +103,7 @@ namespace NProtoBuf { const Reflection& Refl() const { return *Msg.GetReflection(); } - + [[noreturn]] void RaiseUnknown() const { ythrow yexception() << "Unknown field cpp-type: " << (size_t)CppType(); } @@ -127,7 +127,7 @@ namespace NProtoBuf { static TMaybe<TMutableField> ByPath(Message& msg, const TStringBuf& path, bool createPath = false); static TMaybe<TMutableField> ByPath(Message& msg, const TVector<const FieldDescriptor*>& fieldsPath, bool createPath = false); static TMaybe<TMutableField> ByPath(Message& msg, const TFieldPath& fieldsPath, bool createPath = false); - + Message* MutableParent() { return Mut(); } @@ -187,7 +187,7 @@ namespace NProtoBuf { Y_ASSERT(index == 0); return Refl().MutableMessage(Mut(), Fd); } - } + } template <typename TMsg> inline TMsg* AddMessage() { @@ -203,7 +203,7 @@ namespace NProtoBuf { Message* Mut() { return const_cast<Message*>(&Msg); } - + template <typename T> inline void MergeValue(T srcValue); }; @@ -255,8 +255,8 @@ namespace NProtoBuf { template <typename T> inline void TMutableField::MergeValue(T srcValue) { - Add(srcValue); - } + Add(srcValue); + } template <> inline void TMutableField::MergeValue<const Message*>(const Message* srcValue) { @@ -264,8 +264,8 @@ namespace NProtoBuf { Add(srcValue); } else { MutableMessage()->MergeFrom(*srcValue); - } - } + } + } inline void TMutableField::MergeFrom(const TConstField& src) { Y_ASSERT(HasSameType(src)); @@ -286,4 +286,4 @@ namespace NProtoBuf { #undef TMP_MACRO_FOR_CPPTYPE } -} +} diff --git a/library/cpp/protobuf/util/simple_reflection_ut.cpp b/library/cpp/protobuf/util/simple_reflection_ut.cpp index 7647e55db3..169d4703c9 100644 --- a/library/cpp/protobuf/util/simple_reflection_ut.cpp +++ b/library/cpp/protobuf/util/simple_reflection_ut.cpp @@ -1,100 +1,100 @@ -#include "simple_reflection.h" +#include "simple_reflection.h" #include <library/cpp/protobuf/util/ut/sample_for_simple_reflection.pb.h> #include <library/cpp/protobuf/util/ut/extensions.pb.h> - + #include <library/cpp/testing/unittest/registar.h> - -using namespace NProtoBuf; - + +using namespace NProtoBuf; + Y_UNIT_TEST_SUITE(ProtobufSimpleReflection) { static TSample GenSampleForMergeFrom() { TSample smf; - smf.SetOneStr("one str"); - smf.MutableOneMsg()->AddRepInt(1); - smf.AddRepMsg()->AddRepInt(2); - smf.AddRepMsg()->AddRepInt(3); - smf.AddRepStr("one rep str"); - smf.AddRepStr("two rep str"); - smf.SetAnotherOneStr("another one str"); - return smf; - } - + smf.SetOneStr("one str"); + smf.MutableOneMsg()->AddRepInt(1); + smf.AddRepMsg()->AddRepInt(2); + smf.AddRepMsg()->AddRepInt(3); + smf.AddRepStr("one rep str"); + smf.AddRepStr("two rep str"); + smf.SetAnotherOneStr("another one str"); + return smf; + } + Y_UNIT_TEST(MergeFromGeneric) { const TSample src(GenSampleForMergeFrom()); TSample dst; - const Descriptor* descr = dst.GetDescriptor(); - - { - TMutableField dstOneStr(dst, descr->FindFieldByName("OneStr")); - TConstField srcOneStr(src, descr->FindFieldByName("OneStr")); - dstOneStr.MergeFrom(srcOneStr); - UNIT_ASSERT_VALUES_EQUAL(dst.GetOneStr(), src.GetOneStr()); - } - - { // MergeFrom for single message fields acts like a Message::MergeFrom - TMutableField dstOneMsg(dst, descr->FindFieldByName("OneMsg")); - dstOneMsg.MergeFrom(TConstField(src, descr->FindFieldByName("OneMsg"))); - UNIT_ASSERT_VALUES_EQUAL(dst.GetOneMsg().RepIntSize(), src.GetOneMsg().RepIntSize()); - dstOneMsg.MergeFrom(TConstField(src, descr->FindFieldByName("OneMsg"))); + const Descriptor* descr = dst.GetDescriptor(); + + { + TMutableField dstOneStr(dst, descr->FindFieldByName("OneStr")); + TConstField srcOneStr(src, descr->FindFieldByName("OneStr")); + dstOneStr.MergeFrom(srcOneStr); + UNIT_ASSERT_VALUES_EQUAL(dst.GetOneStr(), src.GetOneStr()); + } + + { // MergeFrom for single message fields acts like a Message::MergeFrom + TMutableField dstOneMsg(dst, descr->FindFieldByName("OneMsg")); + dstOneMsg.MergeFrom(TConstField(src, descr->FindFieldByName("OneMsg"))); + UNIT_ASSERT_VALUES_EQUAL(dst.GetOneMsg().RepIntSize(), src.GetOneMsg().RepIntSize()); + dstOneMsg.MergeFrom(TConstField(src, descr->FindFieldByName("OneMsg"))); UNIT_ASSERT_VALUES_EQUAL(dst.GetOneMsg().RepIntSize(), src.GetOneMsg().RepIntSize() * 2); - } - - { // MergeFrom for repeated fields acts like append - TMutableField dstRepMsg(dst, descr->FindFieldByName("RepMsg")); - dstRepMsg.MergeFrom(TConstField(src, descr->FindFieldByName("RepMsg"))); - UNIT_ASSERT_VALUES_EQUAL(dst.RepMsgSize(), src.RepMsgSize()); - dstRepMsg.MergeFrom(TConstField(src, descr->FindFieldByName("RepMsg"))); + } + + { // MergeFrom for repeated fields acts like append + TMutableField dstRepMsg(dst, descr->FindFieldByName("RepMsg")); + dstRepMsg.MergeFrom(TConstField(src, descr->FindFieldByName("RepMsg"))); + UNIT_ASSERT_VALUES_EQUAL(dst.RepMsgSize(), src.RepMsgSize()); + dstRepMsg.MergeFrom(TConstField(src, descr->FindFieldByName("RepMsg"))); UNIT_ASSERT_VALUES_EQUAL(dst.RepMsgSize(), src.RepMsgSize() * 2); - for (size_t repMsgIndex = 0; repMsgIndex < dst.RepMsgSize(); ++repMsgIndex) { - UNIT_ASSERT_VALUES_EQUAL(dst.GetRepMsg(repMsgIndex).RepIntSize(), src.GetRepMsg(0).RepIntSize()); - } - } - } - + for (size_t repMsgIndex = 0; repMsgIndex < dst.RepMsgSize(); ++repMsgIndex) { + UNIT_ASSERT_VALUES_EQUAL(dst.GetRepMsg(repMsgIndex).RepIntSize(), src.GetRepMsg(0).RepIntSize()); + } + } + } + Y_UNIT_TEST(MergeFromSelf) { const TSample sample(GenSampleForMergeFrom()); TSample msg(sample); - const Descriptor* descr = msg.GetDescriptor(); - - TMutableField oneStr(msg, descr->FindFieldByName("OneStr")); - oneStr.MergeFrom(oneStr); - UNIT_ASSERT_VALUES_EQUAL(msg.GetOneStr(), sample.GetOneStr()); - - TMutableField oneMsg(msg, descr->FindFieldByName("OneMsg")); - oneMsg.MergeFrom(oneMsg); // nothing should change - UNIT_ASSERT_VALUES_EQUAL(msg.GetOneMsg().RepIntSize(), sample.GetOneMsg().RepIntSize()); - } - + const Descriptor* descr = msg.GetDescriptor(); + + TMutableField oneStr(msg, descr->FindFieldByName("OneStr")); + oneStr.MergeFrom(oneStr); + UNIT_ASSERT_VALUES_EQUAL(msg.GetOneStr(), sample.GetOneStr()); + + TMutableField oneMsg(msg, descr->FindFieldByName("OneMsg")); + oneMsg.MergeFrom(oneMsg); // nothing should change + UNIT_ASSERT_VALUES_EQUAL(msg.GetOneMsg().RepIntSize(), sample.GetOneMsg().RepIntSize()); + } + Y_UNIT_TEST(MergeFromAnotherFD) { const TSample sample(GenSampleForMergeFrom()); TSample msg(GenSampleForMergeFrom()); - const Descriptor* descr = msg.GetDescriptor(); - - { // string - TMutableField oneStr(msg, descr->FindFieldByName("OneStr")); - TMutableField repStr(msg, descr->FindFieldByName("RepStr")); - TMutableField anotherOneStr(msg, descr->FindFieldByName("AnotherOneStr")); - oneStr.MergeFrom(anotherOneStr); - UNIT_ASSERT_VALUES_EQUAL(msg.GetOneStr(), sample.GetAnotherOneStr()); - oneStr.MergeFrom(repStr); - const size_t sampleRepStrSize = sample.RepStrSize(); - UNIT_ASSERT_VALUES_EQUAL(msg.GetOneStr(), sample.GetRepStr(sampleRepStrSize - 1)); - repStr.MergeFrom(anotherOneStr); - UNIT_ASSERT_VALUES_EQUAL(msg.RepStrSize(), sampleRepStrSize + 1); - UNIT_ASSERT_VALUES_EQUAL(msg.GetRepStr(sampleRepStrSize), msg.GetAnotherOneStr()); - } - - { // Message - TMutableField oneMsg(msg, descr->FindFieldByName("OneMsg")); - TMutableField repMsg(msg, descr->FindFieldByName("RepMsg")); - oneMsg.MergeFrom(repMsg); - const size_t oneMsgRepIntSize = sample.GetOneMsg().RepIntSize(); - const size_t sizeOfAllRepIntsInRepMsg = sample.RepMsgSize(); - UNIT_ASSERT_VALUES_EQUAL(msg.GetOneMsg().RepIntSize(), oneMsgRepIntSize + sizeOfAllRepIntsInRepMsg); - repMsg.MergeFrom(oneMsg); - UNIT_ASSERT_VALUES_EQUAL(msg.RepMsgSize(), sample.RepMsgSize() + 1); - } - } + const Descriptor* descr = msg.GetDescriptor(); + + { // string + TMutableField oneStr(msg, descr->FindFieldByName("OneStr")); + TMutableField repStr(msg, descr->FindFieldByName("RepStr")); + TMutableField anotherOneStr(msg, descr->FindFieldByName("AnotherOneStr")); + oneStr.MergeFrom(anotherOneStr); + UNIT_ASSERT_VALUES_EQUAL(msg.GetOneStr(), sample.GetAnotherOneStr()); + oneStr.MergeFrom(repStr); + const size_t sampleRepStrSize = sample.RepStrSize(); + UNIT_ASSERT_VALUES_EQUAL(msg.GetOneStr(), sample.GetRepStr(sampleRepStrSize - 1)); + repStr.MergeFrom(anotherOneStr); + UNIT_ASSERT_VALUES_EQUAL(msg.RepStrSize(), sampleRepStrSize + 1); + UNIT_ASSERT_VALUES_EQUAL(msg.GetRepStr(sampleRepStrSize), msg.GetAnotherOneStr()); + } + + { // Message + TMutableField oneMsg(msg, descr->FindFieldByName("OneMsg")); + TMutableField repMsg(msg, descr->FindFieldByName("RepMsg")); + oneMsg.MergeFrom(repMsg); + const size_t oneMsgRepIntSize = sample.GetOneMsg().RepIntSize(); + const size_t sizeOfAllRepIntsInRepMsg = sample.RepMsgSize(); + UNIT_ASSERT_VALUES_EQUAL(msg.GetOneMsg().RepIntSize(), oneMsgRepIntSize + sizeOfAllRepIntsInRepMsg); + repMsg.MergeFrom(oneMsg); + UNIT_ASSERT_VALUES_EQUAL(msg.RepMsgSize(), sample.RepMsgSize() + 1); + } + } Y_UNIT_TEST(RemoveByIndex) { TSample msg; @@ -354,6 +354,6 @@ Y_UNIT_TEST_SUITE(ProtobufSimpleReflection) { UNIT_ASSERT(field->HasValue()); UNIT_ASSERT_VALUES_EQUAL(20, field->Get<int>()); } -} +} } } diff --git a/library/cpp/protobuf/util/ut/sample_for_simple_reflection.proto b/library/cpp/protobuf/util/ut/sample_for_simple_reflection.proto index ae714de227..cca1dd869a 100644 --- a/library/cpp/protobuf/util/ut/sample_for_simple_reflection.proto +++ b/library/cpp/protobuf/util/ut/sample_for_simple_reflection.proto @@ -1,15 +1,15 @@ message TInnerSample { - repeated int32 RepInt = 1; + repeated int32 RepInt = 1; extensions 100 to 199; -} - +} + message TSample { - optional string OneStr = 1; + optional string OneStr = 1; optional TInnerSample OneMsg = 2; repeated TInnerSample RepMsg = 3; - repeated string RepStr = 4; - optional string AnotherOneStr = 5; + repeated string RepStr = 4; + optional string AnotherOneStr = 5; optional int32 OneInt = 6; repeated int32 RepInt = 7; @@ -22,4 +22,4 @@ message TSample { repeated EEnum RepEnum = 9; extensions 100 to 199; -} +} diff --git a/library/cpp/protobuf/util/ut/ya.make b/library/cpp/protobuf/util/ut/ya.make index 32c19a87e0..701ba9a8c8 100644 --- a/library/cpp/protobuf/util/ut/ya.make +++ b/library/cpp/protobuf/util/ut/ya.make @@ -5,7 +5,7 @@ UNITTEST_FOR(library/cpp/protobuf/util) SRCS( extensions.proto sample_for_is_equal.proto - sample_for_simple_reflection.proto + sample_for_simple_reflection.proto common_ut.proto pb_io_ut.cpp is_equal_ut.cpp |