intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

1 files changed, 497 insertions, 0 deletions

diff --git a/util/stream/ios_ut.cpp b/util/stream/ios_ut.cpp
new file mode 100644
index 0000000000..139f4296e5
--- /dev/null
+++ b/util/stream/ios_ut.cpp
@@ -0,0 +1,497 @@
+#include "output.h"
+#include "tokenizer.h"
+#include "buffer.h"
+#include "buffered.h"
+#include "walk.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/string/cast.h>
+#include <util/memory/tempbuf.h>
+#include <util/charset/wide.h>
+
+#include <string>
+#include <iostream>
+
+class TStreamsTest: public TTestBase {
+    UNIT_TEST_SUITE(TStreamsTest);
+    UNIT_TEST(TestGenericRead);
+    UNIT_TEST(TestGenericWrite);
+    UNIT_TEST(TestReadLine);
+    UNIT_TEST(TestMemoryStream);
+    UNIT_TEST(TestBufferedIO);
+    UNIT_TEST(TestBufferStream);
+    UNIT_TEST(TestStringStream);
+    UNIT_TEST(TestWtrokaInput);
+    UNIT_TEST(TestStrokaInput);
+    UNIT_TEST(TestReadTo);
+    UNIT_TEST(TestWtrokaOutput);
+    UNIT_TEST(TestIStreamOperators);
+    UNIT_TEST(TestWchar16Output);
+    UNIT_TEST(TestWchar32Output);
+    UNIT_TEST(TestUtf16StingOutputByChars);
+    UNIT_TEST_SUITE_END();
+
+public:
+    void TestGenericRead();
+    void TestGenericWrite();
+    void TestReadLine();
+    void TestMemoryStream();
+    void TestBufferedIO();
+    void TestBufferStream();
+    void TestStringStream();
+    void TestWtrokaInput();
+    void TestStrokaInput();
+    void TestWtrokaOutput();
+    void TestIStreamOperators();
+    void TestReadTo();
+    void TestWchar16Output();
+    void TestWchar32Output();
+    void TestUtf16StingOutputByChars();
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TStreamsTest);
+
+void TStreamsTest::TestIStreamOperators() {
+    TString data("first line\r\nsecond\t\xd1\x82\xd0\xb5\xd1\x81\xd1\x82 line\r\n 1 -4 59 4320000000009999999 c\n -1.5 1e-110");
+    TStringInput si(data);
+
+    TString l1;
+    TString l2;
+    TString l3;
+    TUtf16String w1;
+    TString l4;
+    ui16 i1;
+    i16 i2;
+    i32 i3;
+    ui64 i4;
+    char c1;
+    unsigned char c2;
+    float f1;
+    double f2;
+
+    si >> l1 >> l2 >> l3 >> w1 >> l4 >> i1 >> i2 >> i3 >> i4 >> c1 >> c2 >> f1 >> f2;
+
+    UNIT_ASSERT_EQUAL(l1, "first");
+    UNIT_ASSERT_EQUAL(l2, "line");
+    UNIT_ASSERT_EQUAL(l3, "second");
+    UNIT_ASSERT_EQUAL(l4, "line");
+    UNIT_ASSERT_EQUAL(i1, 1);
+    UNIT_ASSERT_EQUAL(i2, -4);
+    UNIT_ASSERT_EQUAL(i3, 59);
+    UNIT_ASSERT_EQUAL(i4, 4320000000009999999ULL);
+    UNIT_ASSERT_EQUAL(c1, 'c');
+    UNIT_ASSERT_EQUAL(c2, '\n');
+    UNIT_ASSERT_EQUAL(f1, -1.5);
+    UNIT_ASSERT_EQUAL(f2, 1e-110);
+}
+
+void TStreamsTest::TestStringStream() {
+    TStringStream s;
+
+    s << "qw\r\n1234"
+      << "\n"
+      << 34;
+
+    UNIT_ASSERT_EQUAL(s.ReadLine(), "qw");
+    UNIT_ASSERT_EQUAL(s.ReadLine(), "1234");
+
+    s << "\r\n"
+      << 123.1;
+
+    UNIT_ASSERT_EQUAL(s.ReadLine(), "34");
+    UNIT_ASSERT_EQUAL(s.ReadLine(), "123.1");
+
+    UNIT_ASSERT_EQUAL(s.Str(), "qw\r\n1234\n34\r\n123.1");
+
+    // Test stream copying
+    TStringStream sc = s;
+
+    s << "-666-" << 13;
+    sc << "-777-" << 0 << "JackPot";
+
+    UNIT_ASSERT_EQUAL(s.Str(), "qw\r\n1234\n34\r\n123.1-666-13");
+    UNIT_ASSERT_EQUAL(sc.Str(), "qw\r\n1234\n34\r\n123.1-777-0JackPot");
+
+    TStringStream ss;
+    ss = s;
+    s << "... and some trash";
+    UNIT_ASSERT_EQUAL(ss.Str(), "qw\r\n1234\n34\r\n123.1-666-13");
+}
+
+void TStreamsTest::TestGenericRead() {
+    TString s("1234567890");
+    TStringInput si(s);
+    char buf[1024];
+
+    UNIT_ASSERT_EQUAL(si.Read(buf, 6), 6);
+    UNIT_ASSERT_EQUAL(memcmp(buf, "123456", 6), 0);
+    UNIT_ASSERT_EQUAL(si.Read(buf, 6), 4);
+    UNIT_ASSERT_EQUAL(memcmp(buf, "7890", 4), 0);
+}
+
+void TStreamsTest::TestGenericWrite() {
+    TString s;
+    TStringOutput so(s);
+
+    so.Write("123456", 6);
+    so.Write("7890", 4);
+
+    UNIT_ASSERT_EQUAL(s, "1234567890");
+}
+
+void TStreamsTest::TestReadLine() {
+    TString data("1234\r\n5678\nqw");
+    TStringInput si(data);
+
+    UNIT_ASSERT_EQUAL(si.ReadLine(), "1234");
+    UNIT_ASSERT_EQUAL(si.ReadLine(), "5678");
+    UNIT_ASSERT_EQUAL(si.ReadLine(), "qw");
+}
+
+void TStreamsTest::TestMemoryStream() {
+    char buf[1024];
+    TMemoryOutput mo(buf, sizeof(buf));
+    bool ehandled = false;
+
+    try {
+        for (size_t i = 0; i < sizeof(buf) + 1; ++i) {
+            mo.Write(i % 127);
+        }
+    } catch (...) {
+        ehandled = true;
+    }
+
+    UNIT_ASSERT_EQUAL(ehandled, true);
+
+    for (size_t i = 0; i < sizeof(buf); ++i) {
+        UNIT_ASSERT_EQUAL(buf[i], (char)(i % 127));
+    }
+}
+
+class TMyStringOutput: public IOutputStream {
+public:
+    inline TMyStringOutput(TString& s, size_t buflen) noexcept
+        : S_(s)
+        , BufLen_(buflen)
+    {
+    }
+
+    ~TMyStringOutput() override = default;
+
+    void DoWrite(const void* data, size_t len) override {
+        S_.Write(data, len);
+        UNIT_ASSERT(len < BufLen_ || ((len % BufLen_) == 0));
+    }
+
+    void DoWriteV(const TPart* p, size_t count) override {
+        TString s;
+
+        for (size_t i = 0; i < count; ++i) {
+            s.append((const char*)p[i].buf, p[i].len);
+        }
+
+        DoWrite(s.data(), s.size());
+    }
+
+private:
+    TStringOutput S_;
+    const size_t BufLen_;
+};
+
+void TStreamsTest::TestBufferedIO() {
+    TString s;
+
+    {
+        const size_t buflen = 7;
+        TBuffered<TMyStringOutput> bo(buflen, s, buflen);
+
+        for (size_t i = 0; i < 1000; ++i) {
+            TString str(" ");
+            str += ToString(i % 10);
+
+            bo.Write(str.data(), str.size());
+        }
+
+        bo.Finish();
+    }
+
+    UNIT_ASSERT_EQUAL(s.size(), 2000);
+
+    {
+        const size_t buflen = 11;
+        TBuffered<TStringInput> bi(buflen, s);
+
+        for (size_t i = 0; i < 1000; ++i) {
+            TString str(" ");
+            str += ToString(i % 10);
+
+            char buf[3];
+
+            UNIT_ASSERT_EQUAL(bi.Load(buf, 2), 2);
+
+            buf[2] = 0;
+
+            UNIT_ASSERT_EQUAL(str, buf);
+        }
+    }
+
+    s.clear();
+
+    {
+        const size_t buflen = 13;
+        TBuffered<TMyStringOutput> bo(buflen, s, buflen);
+        TString f = "1234567890";
+
+        for (size_t i = 0; i < 10; ++i) {
+            f += f;
+        }
+
+        for (size_t i = 0; i < 1000; ++i) {
+            bo.Write(f.data(), i);
+        }
+
+        bo.Finish();
+    }
+}
+
+void TStreamsTest::TestBufferStream() {
+    TBufferStream stream;
+    TString s = "test";
+
+    stream.Write(s.data(), s.size());
+    char buf[5];
+    size_t bytesRead = stream.Read(buf, 4);
+    UNIT_ASSERT_EQUAL(4, bytesRead);
+    UNIT_ASSERT_EQUAL(0, strncmp(s.data(), buf, 4));
+
+    stream.Write(s.data(), s.size());
+    bytesRead = stream.Read(buf, 2);
+    UNIT_ASSERT_EQUAL(2, bytesRead);
+    UNIT_ASSERT_EQUAL(0, strncmp("te", buf, 2));
+
+    bytesRead = stream.Read(buf, 2);
+    UNIT_ASSERT_EQUAL(2, bytesRead);
+    UNIT_ASSERT_EQUAL(0, strncmp("st", buf, 2));
+
+    bytesRead = stream.Read(buf, 2);
+    UNIT_ASSERT_EQUAL(0, bytesRead);
+}
+
+namespace {
+    class TStringListInput: public IWalkInput {
+    public:
+        TStringListInput(const TVector<TString>& data)
+            : Data_(data)
+            , Index_(0)
+        {
+        }
+
+    protected:
+        size_t DoUnboundedNext(const void** ptr) override {
+            if (Index_ >= Data_.size()) {
+                return 0;
+            }
+
+            const TString& string = Data_[Index_++];
+
+            *ptr = string.data();
+            return string.size();
+        }
+
+    private:
+        const TVector<TString>& Data_;
+        size_t Index_;
+    };
+
+    const char Text[] =
+        // UTF8 encoded "one \ntwo\r\nthree\n\tfour\nfive\n" in russian and ...
+        "один \n"
+        "два\r\n"
+        "три\n"
+        "\tчетыре\n"
+        "пять\n"
+        // ... additional test cases
+        "\r\n"
+        "\n\r" // this char goes to the front of the next string
+        "one two\n"
+        "123\r\n"
+        "\t\r ";
+
+    const char* Expected[] = {
+        // UTF8 encoded "one ", "two", "three", "\tfour", "five" in russian and ...
+        "один ",
+        "два",
+        "три",
+        "\tчетыре",
+        "пять",
+        // ... additional test cases
+        "",
+        "",
+        "\rone two",
+        "123",
+        "\t\r "};
+    void TestStreamReadTo1(IInputStream& input, const char* comment) {
+        TString tmp;
+        input.ReadTo(tmp, 'c');
+        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "111a222b333", comment);
+
+        char tmp2;
+        input.Read(&tmp2, 1);
+        UNIT_ASSERT_VALUES_EQUAL_C(tmp2, '4', comment);
+
+        input.ReadTo(tmp, '6');
+        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "44d555e", comment);
+
+        tmp = input.ReadAll();
+        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "66f", comment);
+    }
+
+    void TestStreamReadTo2(IInputStream& input, const char* comment) {
+        TString s;
+        size_t i = 0;
+        while (input.ReadLine(s)) {
+            UNIT_ASSERT_C(i < Y_ARRAY_SIZE(Expected), comment);
+            UNIT_ASSERT_VALUES_EQUAL_C(s, Expected[i], comment);
+            ++i;
+        }
+    }
+
+    void TestStreamReadTo3(IInputStream& input, const char* comment) {
+        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadLine(), "111a222b333c444d555e666f", comment);
+    }
+
+    void TestStreamReadTo4(IInputStream& input, const char* comment) {
+        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "one", comment);
+        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "two", comment);
+        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "three", comment);
+    }
+
+    void TestStrokaInput(IInputStream& input, const char* comment) {
+        TString line;
+        ui32 i = 0;
+        TInstant start = Now();
+        while (input.ReadLine(line)) {
+            ++i;
+        }
+        Cout << comment << ":" << (Now() - start).SecondsFloat() << Endl;
+        UNIT_ASSERT_VALUES_EQUAL(i, 100000);
+    }
+
+    template <class T>
+    void TestStreamReadTo(const TString& text, T test) {
+        TStringInput is(text);
+        test(is, "TStringInput");
+        TMemoryInput mi(text.data(), text.size());
+        test(mi, "TMemoryInput");
+        TBuffer b(text.data(), text.size());
+        TBufferInput bi(b);
+        test(bi, "TBufferInput");
+        TStringInput slave(text);
+        TBufferedInput bdi(&slave);
+        test(bdi, "TBufferedInput");
+        TVector<TString> lst(1, text);
+        TStringListInput sli(lst);
+        test(sli, "IWalkInput");
+    }
+}
+
+void TStreamsTest::TestReadTo() {
+    TestStreamReadTo("111a222b333c444d555e666f", TestStreamReadTo1);
+    TestStreamReadTo(Text, TestStreamReadTo2);
+    TestStreamReadTo("111a222b333c444d555e666f", TestStreamReadTo3);
+    TString withZero = "one";
+    withZero.append('\0').append("two").append('\0').append("three");
+    TestStreamReadTo(withZero, TestStreamReadTo4);
+}
+
+void TStreamsTest::TestStrokaInput() {
+    TString s;
+    for (ui32 i = 0; i < 100000; ++i) {
+        TVector<char> d(i % 1000, 'a');
+        s.append(d.data(), d.size());
+        s.append('\n');
+    }
+    TestStreamReadTo(s, ::TestStrokaInput);
+}
+
+void TStreamsTest::TestWtrokaInput() {
+    const TString s(Text);
+    TStringInput is(s);
+    TUtf16String w;
+    size_t i = 0;
+
+    while (is.ReadLine(w)) {
+        UNIT_ASSERT(i < Y_ARRAY_SIZE(Expected));
+        UNIT_ASSERT_VALUES_EQUAL(w, UTF8ToWide(Expected[i]));
+
+        ++i;
+    }
+}
+
+void TStreamsTest::TestWtrokaOutput() {
+    TString s;
+    TStringOutput os(s);
+    const size_t n = sizeof(Expected) / sizeof(Expected[0]);
+
+    for (size_t i = 0; i < n; ++i) {
+        TUtf16String w = UTF8ToWide(Expected[i]);
+
+        os << w;
+
+        if (i == 1 || i == 5 || i == 8) {
+            os << '\r';
+        }
+
+        if (i < n - 1) {
+            os << '\n';
+        }
+    }
+
+    UNIT_ASSERT(s == Text);
+}
+
+void TStreamsTest::TestWchar16Output() {
+    TString s;
+    TStringOutput os(s);
+    os << wchar16(97); // latin a
+    os << u'\u044E';   // cyrillic ю
+    os << u'я';
+    os << wchar16(0xD801); // high surrogate is printed as replacement character U+FFFD
+    os << u'b';
+
+    UNIT_ASSERT_VALUES_EQUAL(s, "aюя"
+                                "\xEF\xBF\xBD"
+                                "b");
+}
+
+void TStreamsTest::TestWchar32Output() {
+    TString s;
+    TStringOutput os(s);
+    os << wchar32(97); // latin a
+    os << U'\u044E';   // cyrillic ю
+    os << U'я';
+    os << U'\U0001F600'; // grinning face
+    os << u'b';
+
+    UNIT_ASSERT_VALUES_EQUAL(s, "aюя"
+                                "\xF0\x9F\x98\x80"
+                                "b");
+}
+
+void TStreamsTest::TestUtf16StingOutputByChars() {
+    TString s = "\xd1\x87\xd0\xb8\xd1\x81\xd1\x82\xd0\xb8\xd1\x87\xd0\xb8\xd1\x81\xd1\x82\xd0\xb8";
+    TUtf16String w = UTF8ToWide(s);
+
+    UNIT_ASSERT_VALUES_EQUAL(w.size(), 10);
+
+    TStringStream stream0;
+    stream0 << w;
+    UNIT_ASSERT_VALUES_EQUAL(stream0.Str(), s);
+
+    TStringStream stream1;
+    for (size_t i = 0; i < 10; i++) {
+        stream1 << w[i];
+    }
+    UNIT_ASSERT_VALUES_EQUAL(stream1.Str(), s);
+}