intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

1 files changed, 329 insertions, 0 deletions

diff --git a/library/cpp/on_disk/chunks/chunks_ut.cpp b/library/cpp/on_disk/chunks/chunks_ut.cpp
new file mode 100644
index 0000000000..f727647f7f
--- /dev/null
+++ b/library/cpp/on_disk/chunks/chunks_ut.cpp
@@ -0,0 +1,329 @@
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/file.h>
+#include <util/system/filemap.h>
+#include <util/system/tempfile.h>
+
+#include "chunked_helpers.h"
+
+/// Data for TChunkedHelpersTest::TestGeneralVector
+struct TPodStruct {
+    int x;
+    float y;
+    TPodStruct(int _x = 0, float _y = 0)
+        : x(_x)
+        , y(_y)
+    {
+    }
+};
+/// And its serialization
+template <>
+struct TSaveLoadVectorNonPodElement<TPodStruct> {
+    typedef TPodStruct TItem;
+    static inline void Save(IOutputStream* out, const TItem& item) {
+        TSerializer<int>::Save(out, item.x);
+        TSerializer<float>::Save(out, item.y);
+    }
+
+    static inline void Load(IInputStream* in, TItem& item, size_t elementSize) {
+        Y_ASSERT(elementSize == sizeof(TItem));
+        TSerializer<int>::Load(in, item.x);
+        TSerializer<float>::Load(in, item.y);
+    }
+};
+
+class TChunkedHelpersTest: public TTestBase {
+    UNIT_TEST_SUITE(TChunkedHelpersTest);
+    UNIT_TEST(TestHash)
+    UNIT_TEST(TestGeneralVector)
+    UNIT_TEST(TestStrings);
+    UNIT_TEST(TestNamedChunkedData);
+    UNIT_TEST_SUITE_END();
+
+public:
+    void TestHash() {
+        {
+            TBufferStream stream;
+            {
+                TPlainHashWriter<ui64, ui16> writer;
+                writer.Add(5, 7);
+                writer.Save(stream);
+            }
+
+            {
+                TBlob temp = TBlob::FromStreamSingleThreaded(stream);
+                TPlainHash<ui64, ui16> reader(temp);
+                ui16 value = 0;
+                UNIT_ASSERT(reader.Find(5, &value));
+                UNIT_ASSERT_EQUAL(7, value);
+                UNIT_ASSERT(!reader.Find(6, &value));
+            }
+        }
+
+        {
+            TBufferStream stream;
+            int v = 1;
+            wchar16 k = 'a';
+            {
+                TPlainHashWriter<wchar16, void*> writer;
+                writer.Add(k, &v);
+                writer.Save(stream);
+            }
+            {
+                TBlob temp = TBlob::FromStreamSingleThreaded(stream);
+                TPlainHash<wchar16, void*> reader(temp);
+                void* value = nullptr;
+                UNIT_ASSERT(reader.Find(k, &value));
+                UNIT_ASSERT_EQUAL((int*)value, &v);
+            }
+        }
+    }
+
+    void TestGeneralVector() {
+        { /// ui32
+            const size_t N = 3;
+            TBufferStream stream;
+            {
+                TGeneralVectorWriter<ui32> writer;
+                for (size_t i = 0; i < N; ++i)
+                    writer.PushBack(i);
+                writer.Save(stream);
+            }
+            {
+                TBlob temp = TBlob::FromStreamSingleThreaded(stream);
+                TGeneralVector<ui32> reader(temp);
+                UNIT_ASSERT_EQUAL(reader.GetSize(), N);
+                for (size_t i = 0; i < N; ++i) {
+                    ui32 value;
+                    reader.Get(i, value);
+                    UNIT_ASSERT_EQUAL(value, i);
+                    UNIT_ASSERT_EQUAL(reader.At(i), i);
+                }
+                UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(ui32));
+            }
+        }
+        { /// TString
+            const size_t N = 4;
+            TBufferStream stream;
+            {
+                TGeneralVectorWriter<TString> writer;
+                for (size_t i = 0; i < N; ++i)
+                    writer.PushBack(ToString(i));
+                writer.Save(stream);
+            }
+            {
+                TBlob temp = TBlob::FromStreamSingleThreaded(stream);
+                TGeneralVector<TString> reader(temp);
+                UNIT_ASSERT_EQUAL(reader.GetSize(), N);
+                for (size_t i = 0; i < N; ++i) {
+                    TString value;
+                    reader.Get(i, value);
+                    UNIT_ASSERT_EQUAL(value, ToString(i));
+                    UNIT_ASSERT_EQUAL(reader.Get(i), ToString(i));
+                }
+                UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * 2);
+            }
+        }
+        { /// some other struct
+            typedef TPodStruct TItem;
+            const size_t N = 2;
+            TBufferStream stream;
+            {
+                TGeneralVectorWriter<TItem> writer;
+                writer.PushBack(TItem(1, 2));
+                writer.PushBack(TItem(3, 4));
+                writer.Save(stream);
+            }
+            {
+                TBlob temp = TBlob::FromStreamSingleThreaded(stream);
+                TGeneralVector<TItem> reader(temp);
+                UNIT_ASSERT_EQUAL(reader.GetSize(), N);
+
+                TItem value;
+                reader.Get(0, value);
+                UNIT_ASSERT(value.x == 1 && value.y == 2.0);
+
+                reader.Get(1, value);
+                UNIT_ASSERT(value.x == 3 && value.y == 4.0);
+
+                UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * sizeof(TItem));
+            }
+        }
+        { /// pointer
+            const size_t N = 3;
+            TVector<int> data_holder(N);
+            int* a = &(data_holder[0]);
+            TBufferStream stream;
+            {
+                TGeneralVectorWriter<int*> writer;
+                for (size_t i = 0; i < N; ++i) {
+                    a[i] = i;
+                    writer.PushBack(a + i);
+                }
+                writer.Save(stream);
+            }
+            {
+                TBlob temp = TBlob::FromStreamSingleThreaded(stream);
+                TGeneralVector<int*> reader(temp);
+                UNIT_ASSERT_EQUAL(reader.GetSize(), N);
+                for (size_t i = 0; i < N; ++i) {
+                    int* value;
+                    reader.Get(i, value);
+                    UNIT_ASSERT_EQUAL(value, a + i);
+                    UNIT_ASSERT_EQUAL(reader.At(i), a + i);
+                }
+                UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(int*));
+            }
+        }
+        { /// std::pair<int, int>
+            typedef std::pair<int, int> TItem;
+            const size_t N = 3;
+            TBufferStream stream;
+            {
+                TGeneralVectorWriter<TItem> writer;
+                for (size_t i = 0; i < N; ++i)
+                    writer.PushBack(TItem(i, i));
+                writer.Save(stream);
+            }
+            {
+                TBlob temp = TBlob::FromStreamSingleThreaded(stream);
+                TGeneralVector<TItem> reader(temp);
+                UNIT_ASSERT_EQUAL(reader.GetSize(), N);
+                for (size_t i = 0; i < N; ++i) {
+                    TItem value;
+                    reader.Get(i, value);
+                    UNIT_ASSERT_EQUAL(value, TItem(i, i));
+                }
+                UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(TItem));
+            }
+        }
+    }
+
+    void TestStrings() {
+        const TString FILENAME = "chunked_helpers_test.bin";
+        TTempFileHandle file(FILENAME.c_str());
+
+        {
+            TFixedBufferFileOutput fOut(FILENAME);
+            TStringsVectorWriter stringsWriter;
+            stringsWriter.PushBack("");
+            stringsWriter.PushBack("test");
+            TChunkedDataWriter writer(fOut);
+            WriteBlock(writer, stringsWriter);
+            writer.WriteFooter();
+        }
+
+        {
+            TBlob fIn = TBlob::FromFileSingleThreaded(FILENAME);
+            TStringsVector vct(GetBlock(fIn, 0));
+            UNIT_ASSERT_EQUAL(vct.Get(0), "");
+            UNIT_ASSERT_EQUAL(vct.Get(1), "test");
+
+            bool wasException = false;
+            try {
+                vct.Get(2);
+            } catch (...) {
+                wasException = true;
+            }
+            UNIT_ASSERT(wasException);
+        }
+    }
+
+    void TestNamedChunkedData() {
+        const TString filename = MakeTempName(nullptr, "named_chunked_data_test");
+        TTempFile file(filename);
+
+        {
+            TFixedBufferFileOutput fOut(filename);
+            TNamedChunkedDataWriter writer(fOut);
+
+            writer.NewBlock("alpha");
+            writer.Write("123456");
+
+            writer.NewBlock();
+            writer.Write("anonymous");
+
+            writer.NewBlock("omega");
+            writer.Write("12345678901234567");
+
+            writer.WriteFooter();
+        }
+
+        {
+            TBlob mf = TBlob::FromFileSingleThreaded(filename);
+            TNamedChunkedDataReader reader(mf);
+
+            UNIT_ASSERT(reader.GetBlocksCount() == 3);
+
+            UNIT_ASSERT(reader.HasBlock("alpha"));
+            UNIT_ASSERT(reader.HasBlock("omega"));
+
+            UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(0), "alpha");
+            UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(1), "");
+            UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(2), "omega");
+
+            UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("alpha"), 6); // padding not included
+            UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("omega"), 17);
+
+            UNIT_ASSERT(memcmp(reader.GetBlockByName("alpha"), "123456", 6) == 0);
+            UNIT_ASSERT(memcmp(reader.GetBlock(1), "anonymous", 9) == 0);
+            UNIT_ASSERT(memcmp(reader.GetBlockByName("omega"), "12345678901234567", 17) == 0);
+        }
+    }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TChunkedHelpersTest);
+
+class TChunkedDataTest: public TTestBase {
+private:
+    UNIT_TEST_SUITE(TChunkedDataTest);
+    UNIT_TEST(Test)
+    UNIT_TEST(TestEmpty)
+    UNIT_TEST_SUITE_END();
+
+    void Test() {
+        TBuffer buffer;
+        {
+            TBufferOutput out(buffer);
+            TChunkedDataWriter writer(out);
+            writer.NewBlock();
+            writer << "test";
+            writer.NewBlock();
+            writer << 4;
+            writer.NewBlock();
+            writer.NewBlock();
+            writer << 1;
+            writer << 2;
+            writer.WriteFooter();
+        }
+        {
+            TBlob blob = TBlob::FromBufferSingleThreaded(buffer);
+            TChunkedDataReader data(blob);
+            // printf("%d\n", (int)data.GetBlockLen(3));
+            UNIT_ASSERT_EQUAL(4, data.GetBlockLen(0));
+            UNIT_ASSERT_EQUAL(1, data.GetBlockLen(1));
+            UNIT_ASSERT_EQUAL(0, data.GetBlockLen(2));
+            UNIT_ASSERT_EQUAL(2, data.GetBlockLen(3));
+        }
+    }
+
+    void TestEmpty() {
+        TBuffer buffer;
+        {
+            TBufferOutput out(buffer);
+            TChunkedDataWriter writer(out);
+            writer.NewBlock();
+            writer.NewBlock();
+            writer.WriteFooter();
+        }
+        {
+            TBlob blob = TBlob::FromBufferSingleThreaded(buffer);
+            TChunkedDataReader data(blob);
+            // printf("%d\n", (int)data.GetBlockLen(1));
+            UNIT_ASSERT_EQUAL(0, data.GetBlockLen(0));
+            UNIT_ASSERT_EQUAL(0, data.GetBlockLen(1));
+        }
+    }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TChunkedDataTest);