path: root/util/stream/ios_ut.cpp

#include "output.h"
#include "tokenizer.h" 
#include "buffer.h"
#include "buffered.h"
#include "walk.h"
 
#include <library/cpp/testing/unittest/registar.h>
 
#include <util/string/cast.h> 
#include <util/memory/tempbuf.h>
#include <util/charset/wide.h> 
 
#include <string> 
#include <iostream> 
 
class TStreamsTest: public TTestBase { 
    UNIT_TEST_SUITE(TStreamsTest); 
    UNIT_TEST(TestGenericRead); 
    UNIT_TEST(TestGenericWrite); 
    UNIT_TEST(TestReadLine); 
    UNIT_TEST(TestMemoryStream); 
    UNIT_TEST(TestBufferedIO); 
    UNIT_TEST(TestBufferStream); 
    UNIT_TEST(TestStringStream); 
    UNIT_TEST(TestWtrokaInput); 
    UNIT_TEST(TestStrokaInput);
    UNIT_TEST(TestReadTo);
    UNIT_TEST(TestWtrokaOutput); 
    UNIT_TEST(TestIStreamOperators); 
    UNIT_TEST(TestWchar16Output);
    UNIT_TEST(TestWchar32Output);
    UNIT_TEST(TestUtf16StingOutputByChars);
    UNIT_TEST_SUITE_END(); 
 
public: 
    void TestGenericRead(); 
    void TestGenericWrite(); 
    void TestReadLine(); 
    void TestMemoryStream(); 
    void TestBufferedIO(); 
    void TestBufferStream(); 
    void TestStringStream(); 
    void TestWtrokaInput(); 
    void TestStrokaInput();
    void TestWtrokaOutput(); 
    void TestIStreamOperators(); 
    void TestReadTo();
    void TestWchar16Output();
    void TestWchar32Output();
    void TestUtf16StingOutputByChars();
}; 
 
UNIT_TEST_SUITE_REGISTRATION(TStreamsTest); 
 
void TStreamsTest::TestIStreamOperators() {
    TString data("first line\r\nsecond\t\xd1\x82\xd0\xb5\xd1\x81\xd1\x82 line\r\n 1 -4 59 4320000000009999999 c\n -1.5 1e-110");
    TStringInput si(data);

    TString l1;
    TString l2;
    TString l3;
    TUtf16String w1;
    TString l4;
    ui16 i1;
    i16 i2;
    i32 i3;
    ui64 i4;
    char c1;
    unsigned char c2;
    float f1;
    double f2;

    si >> l1 >> l2 >> l3 >> w1 >> l4 >> i1 >> i2 >> i3 >> i4 >> c1 >> c2 >> f1 >> f2;

    UNIT_ASSERT_EQUAL(l1, "first");
    UNIT_ASSERT_EQUAL(l2, "line");
    UNIT_ASSERT_EQUAL(l3, "second");
    UNIT_ASSERT_EQUAL(l4, "line");
    UNIT_ASSERT_EQUAL(i1, 1);
    UNIT_ASSERT_EQUAL(i2, -4);
    UNIT_ASSERT_EQUAL(i3, 59);
    UNIT_ASSERT_EQUAL(i4, 4320000000009999999ULL);
    UNIT_ASSERT_EQUAL(c1, 'c');
    UNIT_ASSERT_EQUAL(c2, '\n');
    UNIT_ASSERT_EQUAL(f1, -1.5);
    UNIT_ASSERT_EQUAL(f2, 1e-110);
}

void TStreamsTest::TestStringStream() { 
    TStringStream s; 
 
    s << "qw\r\n1234" 
      << "\n" 
      << 34; 
 
    UNIT_ASSERT_EQUAL(s.ReadLine(), "qw"); 
    UNIT_ASSERT_EQUAL(s.ReadLine(), "1234"); 
 
    s << "\r\n" 
      << 123.1; 
 
    UNIT_ASSERT_EQUAL(s.ReadLine(), "34"); 
    UNIT_ASSERT_EQUAL(s.ReadLine(), "123.1");
 
    UNIT_ASSERT_EQUAL(s.Str(), "qw\r\n1234\n34\r\n123.1");

    // Test stream copying
    TStringStream sc = s;

    s << "-666-" << 13;
    sc << "-777-" << 0 << "JackPot";

    UNIT_ASSERT_EQUAL(s.Str(), "qw\r\n1234\n34\r\n123.1-666-13");
    UNIT_ASSERT_EQUAL(sc.Str(), "qw\r\n1234\n34\r\n123.1-777-0JackPot");

    TStringStream ss;
    ss = s;
    s << "... and some trash";
    UNIT_ASSERT_EQUAL(ss.Str(), "qw\r\n1234\n34\r\n123.1-666-13");
} 
 
void TStreamsTest::TestGenericRead() { 
    TString s("1234567890");
    TStringInput si(s); 
    char buf[1024]; 
 
    UNIT_ASSERT_EQUAL(si.Read(buf, 6), 6); 
    UNIT_ASSERT_EQUAL(memcmp(buf, "123456", 6), 0); 
    UNIT_ASSERT_EQUAL(si.Read(buf, 6), 4); 
    UNIT_ASSERT_EQUAL(memcmp(buf, "7890", 4), 0); 
} 
 
void TStreamsTest::TestGenericWrite() { 
    TString s;
    TStringOutput so(s); 
 
    so.Write("123456", 6); 
    so.Write("7890", 4); 
 
    UNIT_ASSERT_EQUAL(s, "1234567890"); 
} 
 
void TStreamsTest::TestReadLine() { 
    TString data("1234\r\n5678\nqw");
    TStringInput si(data); 
 
    UNIT_ASSERT_EQUAL(si.ReadLine(), "1234"); 
    UNIT_ASSERT_EQUAL(si.ReadLine(), "5678"); 
    UNIT_ASSERT_EQUAL(si.ReadLine(), "qw"); 
} 
 
void TStreamsTest::TestMemoryStream() { 
    char buf[1024]; 
    TMemoryOutput mo(buf, sizeof(buf)); 
    bool ehandled = false; 
 
    try { 
        for (size_t i = 0; i < sizeof(buf) + 1; ++i) { 
            mo.Write(i % 127); 
        } 
    } catch (...) { 
        ehandled = true; 
    } 
 
    UNIT_ASSERT_EQUAL(ehandled, true); 
 
    for (size_t i = 0; i < sizeof(buf); ++i) { 
        UNIT_ASSERT_EQUAL(buf[i], (char)(i % 127)); 
    } 
} 
 
class TMyStringOutput: public IOutputStream {
public: 
    inline TMyStringOutput(TString& s, size_t buflen) noexcept
        : S_(s) 
        , BufLen_(buflen) 
    { 
    } 
 
    ~TMyStringOutput() override = default;
 
    void DoWrite(const void* data, size_t len) override {
        S_.Write(data, len); 
        UNIT_ASSERT(len < BufLen_ || ((len % BufLen_) == 0)); 
    } 
 
    void DoWriteV(const TPart* p, size_t count) override {
        TString s;
 
        for (size_t i = 0; i < count; ++i) { 
            s.append((const char*)p[i].buf, p[i].len); 
        } 
 
        DoWrite(s.data(), s.size());
    } 
 
private: 
    TStringOutput S_; 
    const size_t BufLen_; 
}; 
 
void TStreamsTest::TestBufferedIO() { 
    TString s;
 
    { 
        const size_t buflen = 7; 
        TBuffered<TMyStringOutput> bo(buflen, s, buflen); 
 
        for (size_t i = 0; i < 1000; ++i) { 
            TString str(" ");
            str += ToString(i % 10); 
 
            bo.Write(str.data(), str.size());
        } 
 
        bo.Finish(); 
    } 
 
    UNIT_ASSERT_EQUAL(s.size(), 2000);
 
    { 
        const size_t buflen = 11; 
        TBuffered<TStringInput> bi(buflen, s); 
 
        for (size_t i = 0; i < 1000; ++i) { 
            TString str(" ");
            str += ToString(i % 10); 
 
            char buf[3]; 
 
            UNIT_ASSERT_EQUAL(bi.Load(buf, 2), 2); 
 
            buf[2] = 0; 
 
            UNIT_ASSERT_EQUAL(str, buf); 
        } 
    } 
 
    s.clear(); 
 
    { 
        const size_t buflen = 13; 
        TBuffered<TMyStringOutput> bo(buflen, s, buflen); 
        TString f = "1234567890";
 
        for (size_t i = 0; i < 10; ++i) { 
            f += f; 
        } 
 
        for (size_t i = 0; i < 1000; ++i) { 
            bo.Write(f.data(), i);
        } 
 
        bo.Finish(); 
    } 
} 

void TStreamsTest::TestBufferStream() { 
    TBufferStream stream;
    TString s = "test";
 
    stream.Write(s.data(), s.size());
    char buf[5];
    size_t bytesRead = stream.Read(buf, 4);
    UNIT_ASSERT_EQUAL(4, bytesRead);
    UNIT_ASSERT_EQUAL(0, strncmp(s.data(), buf, 4));

    stream.Write(s.data(), s.size());
    bytesRead = stream.Read(buf, 2);
    UNIT_ASSERT_EQUAL(2, bytesRead);
    UNIT_ASSERT_EQUAL(0, strncmp("te", buf, 2));
 
    bytesRead = stream.Read(buf, 2);
    UNIT_ASSERT_EQUAL(2, bytesRead);
    UNIT_ASSERT_EQUAL(0, strncmp("st", buf, 2));

    bytesRead = stream.Read(buf, 2);
    UNIT_ASSERT_EQUAL(0, bytesRead);
}

namespace {
    class TStringListInput: public IWalkInput {
    public:
        TStringListInput(const TVector<TString>& data)
            : Data_(data)
            , Index_(0)
        {
        }

    protected:
        size_t DoUnboundedNext(const void** ptr) override {
            if (Index_ >= Data_.size()) {
                return 0;
            }

            const TString& string = Data_[Index_++];

            *ptr = string.data();
            return string.size();
        }

    private:
        const TVector<TString>& Data_;
        size_t Index_;
    };

    const char Text[] =
        // UTF8 encoded "one \ntwo\r\nthree\n\tfour\nfive\n" in russian and ...
        "один \n"
        "два\r\n"
        "три\n"
        "\tчетыре\n"
        "пять\n"
        // ... additional test cases
        "\r\n"
        "\n\r" // this char goes to the front of the next string
        "one two\n"
        "123\r\n"
        "\t\r ";

    const char* Expected[] = {
        // UTF8 encoded "one ", "two", "three", "\tfour", "five" in russian and ...
        "один ",
        "два",
        "три",
        "\tчетыре",
        "пять",
        // ... additional test cases
        "",
        "",
        "\rone two",
        "123",
        "\t\r "}; 
    void TestStreamReadTo1(IInputStream& input, const char* comment) {
        TString tmp;
        input.ReadTo(tmp, 'c'); 
        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "111a222b333", comment); 

        char tmp2; 
        input.Read(&tmp2, 1); 
        UNIT_ASSERT_VALUES_EQUAL_C(tmp2, '4', comment); 

        input.ReadTo(tmp, '6'); 
        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "44d555e", comment); 

        tmp = input.ReadAll(); 
        UNIT_ASSERT_VALUES_EQUAL_C(tmp, "66f", comment); 
    } 

    void TestStreamReadTo2(IInputStream& input, const char* comment) {
        TString s;
        size_t i = 0; 
        while (input.ReadLine(s)) { 
            UNIT_ASSERT_C(i < Y_ARRAY_SIZE(Expected), comment); 
            UNIT_ASSERT_VALUES_EQUAL_C(s, Expected[i], comment); 
            ++i; 
        }
    } 

    void TestStreamReadTo3(IInputStream& input, const char* comment) {
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadLine(), "111a222b333c444d555e666f", comment); 
    } 

    void TestStreamReadTo4(IInputStream& input, const char* comment) {
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "one", comment); 
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "two", comment); 
        UNIT_ASSERT_VALUES_EQUAL_C(input.ReadTo('\0'), "three", comment); 
    } 

    void TestStrokaInput(IInputStream& input, const char* comment) {
        TString line;
        ui32 i = 0; 
        TInstant start = Now(); 
        while (input.ReadLine(line)) { 
            ++i; 
        }
        Cout << comment << ":" << (Now() - start).SecondsFloat() << Endl; 
        UNIT_ASSERT_VALUES_EQUAL(i, 100000); 
    } 

    template <class T> 
    void TestStreamReadTo(const TString& text, T test) {
        TStringInput is(text); 
        test(is, "TStringInput"); 
        TMemoryInput mi(text.data(), text.size());
        test(mi, "TMemoryInput"); 
        TBuffer b(text.data(), text.size());
        TBufferInput bi(b); 
        test(bi, "TBufferInput"); 
        TStringInput slave(text); 
        TBufferedInput bdi(&slave); 
        test(bdi, "TBufferedInput"); 
        TVector<TString> lst(1, text);
        TStringListInput sli(lst); 
        test(sli, "IWalkInput");
    } 
}

void TStreamsTest::TestReadTo() {
    TestStreamReadTo("111a222b333c444d555e666f", TestStreamReadTo1);
    TestStreamReadTo(Text, TestStreamReadTo2);
    TestStreamReadTo("111a222b333c444d555e666f", TestStreamReadTo3);
    TString withZero = "one";
    withZero.append('\0').append("two").append('\0').append("three");
    TestStreamReadTo(withZero, TestStreamReadTo4);
}

void TStreamsTest::TestStrokaInput() {
    TString s;
    for (ui32 i = 0; i < 100000; ++i) {
        TVector<char> d(i % 1000, 'a');
        s.append(d.data(), d.size());
        s.append('\n');
    }
    TestStreamReadTo(s, ::TestStrokaInput); 
}

void TStreamsTest::TestWtrokaInput() {
    const TString s(Text);
    TStringInput is(s);
    TUtf16String w;
    size_t i = 0; 

    while (is.ReadLine(w)) {
        UNIT_ASSERT(i < Y_ARRAY_SIZE(Expected));
        UNIT_ASSERT_VALUES_EQUAL(w, UTF8ToWide(Expected[i])); 

        ++i;
    }
}

void TStreamsTest::TestWtrokaOutput() {
    TString s;
    TStringOutput os(s);
    const size_t n = sizeof(Expected) / sizeof(Expected[0]);

    for (size_t i = 0; i < n; ++i) {
        TUtf16String w = UTF8ToWide(Expected[i]);

        os << w;
 
        if (i == 1 || i == 5 || i == 8) { 
            os << '\r';
        } 
 
        if (i < n - 1) { 
            os << '\n';
        } 
    }

    UNIT_ASSERT(s == Text);
}

void TStreamsTest::TestWchar16Output() {
    TString s;
    TStringOutput os(s);
    os << wchar16(97); // latin a
    os << u'\u044E';   // cyrillic ю 
    os << u'я';
    os << wchar16(0xD801); // high surrogate is printed as replacement character U+FFFD
    os << u'b';

    UNIT_ASSERT_VALUES_EQUAL(s, "aюя" 
                                "\xEF\xBF\xBD" 
                                "b"); 
}

void TStreamsTest::TestWchar32Output() {
    TString s;
    TStringOutput os(s);
    os << wchar32(97); // latin a
    os << U'\u044E';   // cyrillic ю 
    os << U'я';
    os << U'\U0001F600'; // grinning face
    os << u'b';

    UNIT_ASSERT_VALUES_EQUAL(s, "aюя" 
                                "\xF0\x9F\x98\x80" 
                                "b"); 
}

void TStreamsTest::TestUtf16StingOutputByChars() {
    TString s = "\xd1\x87\xd0\xb8\xd1\x81\xd1\x82\xd0\xb8\xd1\x87\xd0\xb8\xd1\x81\xd1\x82\xd0\xb8";
    TUtf16String w = UTF8ToWide(s);

    UNIT_ASSERT_VALUES_EQUAL(w.size(), 10);

    TStringStream stream0;
    stream0 << w;
    UNIT_ASSERT_VALUES_EQUAL(stream0.Str(), s);

    TStringStream stream1;
    for (size_t i = 0; i < 10; i++) {
        stream1 << w[i];
    }
    UNIT_ASSERT_VALUES_EQUAL(stream1.Str(), s);
}