intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

1 files changed, 382 insertions, 0 deletions

diff --git a/library/cpp/yson_pull/ut/loop_ut.cpp b/library/cpp/yson_pull/ut/loop_ut.cpp
new file mode 100644
index 0000000000..8c7b11dd1c
--- /dev/null
+++ b/library/cpp/yson_pull/ut/loop_ut.cpp
@@ -0,0 +1,382 @@
+#include <library/cpp/yson_pull/input.h>
+#include <library/cpp/yson_pull/output.h>
+#include <library/cpp/yson_pull/reader.h>
+#include <library/cpp/yson_pull/writer.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <cerrno>
+#include <cmath>
+
+#ifdef _unix_
+#include <unistd.h>
+#include <sys/wait.h>
+#endif
+
+namespace {
+    constexpr const char* alphabet =
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+    void generate(NYsonPull::TWriter& writer, size_t count) {
+        writer.BeginStream();
+        for (size_t i = 0; i < count; ++i) {
+            writer.BeginMap()
+                .Key("ints")
+                .BeginList()
+                .Int64(0)
+                .Int64(-1)
+                .Int64(1000)
+                .Int64(-1000)
+                .EndList()
+                .Key("uints")
+                .BeginList()
+                .UInt64(0)
+                .UInt64(1000)
+                .UInt64(10000000)
+                .EndList()
+                .Key("entities")
+                .BeginList()
+                .Entity()
+                .BeginAttributes()
+                .Key("color")
+                .String("blue")
+                .Key("size")
+                .Int64(100)
+                .EndAttributes()
+                .Entity()
+                .Entity()
+                .EndList()
+                .Key("booleans")
+                .BeginList()
+                .Boolean(true)
+                .Boolean(false)
+                .Boolean(true)
+                .EndList()
+                .Key("floats")
+                .BeginList()
+                .Float64(0.0)
+                .Float64(13.0e30)
+                .Float64(M_PI)
+                .EndList()
+                .Key("strings")
+                .BeginList()
+                .String("hello")
+                .String("")
+                .String("foo \"-bar-\" baz")
+                .String("oh\nwow")
+                .String(alphabet)
+                .EndList()
+                .EndMap();
+        }
+        writer.EndStream();
+    }
+
+#ifdef __clang__
+    // XXX: With all the macros below (esp. UNIT_ASSERT_VALUES_EQUAL) unfolded,
+    // the time it takes clang to optimize generated code becomes abysmal.
+    // Locally disabling optimization brings it back to normal.
+    __attribute__((optnone))
+#endif // __clang__
+    void
+    verify(NYsonPull::TReader& reader, size_t count) {
+#define NEXT(name__) \
+    {                \
+        auto& name__ = reader.NextEvent(); // SCOPED_TRACE(e);
+#define END_NEXT }
+#define NEXT_TYPE(type__)                                                  \
+    NEXT(e) {                                                              \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::type__, e.Type()); \
+    }                                                                      \
+    END_NEXT
+#define NEXT_KEY(key__)                                                 \
+    NEXT(e) {                                                           \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); \
+        UNIT_ASSERT_VALUES_EQUAL(key__, e.AsString());                  \
+    }                                                                   \
+    END_NEXT
+#define NEXT_SCALAR(type__, value__)                                                   \
+    NEXT(e) {                                                                          \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type());             \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::type__, e.AsScalar().Type()); \
+        UNIT_ASSERT_VALUES_EQUAL(value__, e.AsScalar().As##type__());                  \
+    }                                                                                  \
+    END_NEXT
+#define NEXT_ENTITY()                                                                  \
+    NEXT(e) {                                                                          \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type());             \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::Entity, e.AsScalar().Type()); \
+    }                                                                                  \
+    END_NEXT
+#define NEXT_FLOAT64(value__)                                                           \
+    NEXT(e) {                                                                           \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type());              \
+        UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::Float64, e.AsScalar().Type()); \
+        UNIT_ASSERT_DOUBLES_EQUAL(value__, e.AsScalar().AsFloat64(), 1e-5);             \
+    }                                                                                   \
+    END_NEXT
+
+        constexpr auto true_ = true;
+        constexpr auto false_ = false;
+
+        NEXT_TYPE(BeginStream);
+        for (size_t i = 0; i < count; ++i) {
+            NEXT_TYPE(BeginMap);
+            NEXT_KEY("ints") {
+                NEXT_TYPE(BeginList);
+                NEXT_SCALAR(Int64, 0);
+                NEXT_SCALAR(Int64, -1);
+                NEXT_SCALAR(Int64, 1000);
+                NEXT_SCALAR(Int64, -1000);
+                NEXT_TYPE(EndList);
+            }
+            NEXT_KEY("uints") {
+                NEXT_TYPE(BeginList);
+                NEXT_SCALAR(UInt64, 0U);
+                NEXT_SCALAR(UInt64, 1000U);
+                NEXT_SCALAR(UInt64, 10000000U);
+                NEXT_TYPE(EndList);
+            }
+            NEXT_KEY("entities") {
+                NEXT_TYPE(BeginList);
+                NEXT_ENTITY();
+                NEXT_TYPE(BeginAttributes) {
+                    NEXT_KEY("color") {
+                        NEXT_SCALAR(String, "blue");
+                    }
+                    NEXT_KEY("size") {
+                        NEXT_SCALAR(Int64, 100);
+                    }
+                }
+                NEXT_TYPE(EndAttributes);
+                NEXT_ENTITY();
+                NEXT_ENTITY();
+                NEXT_TYPE(EndList);
+            }
+            NEXT_KEY("booleans") {
+                NEXT_TYPE(BeginList);
+                NEXT_SCALAR(Boolean, true_);
+                NEXT_SCALAR(Boolean, false_);
+                NEXT_SCALAR(Boolean, true_);
+                NEXT_TYPE(EndList);
+            }
+            NEXT_KEY("floats") {
+                NEXT_TYPE(BeginList);
+                NEXT_FLOAT64(0.0);
+                NEXT_FLOAT64(13.0e30);
+                NEXT_FLOAT64(M_PI);
+                NEXT_TYPE(EndList);
+            }
+            NEXT_KEY("strings") {
+                NEXT_TYPE(BeginList);
+                NEXT_SCALAR(String, "hello");
+                NEXT_SCALAR(String, "");
+                NEXT_SCALAR(String, "foo \"-bar-\" baz");
+                NEXT_SCALAR(String, "oh\nwow");
+                NEXT_SCALAR(String, alphabet);
+                NEXT_TYPE(EndList);
+            }
+            NEXT_TYPE(EndMap);
+        }
+        NEXT_TYPE(EndStream);
+
+#undef NEXT
+#undef END_NEXT
+#undef NEXT_TYPE
+#undef NEXT_KEY
+#undef NEXT_SCALAR
+    }
+
+    class sys_error {};
+
+    IOutputStream& operator<<(IOutputStream& stream, const sys_error&) {
+        stream << strerror(errno);
+        return stream;
+    }
+
+    NYsonPull::TReader make_reader(THolder<NYsonPull::NInput::IStream> stream) {
+        return NYsonPull::TReader(
+            std::move(stream),
+            NYsonPull::EStreamType::ListFragment);
+    }
+
+    template <typename Function>
+    void test_memory(Function make_writer, size_t nrepeat) {
+        TString text;
+        {
+            auto writer = make_writer(NYsonPull::NOutput::FromString(&text));
+            generate(writer, nrepeat);
+        }
+        {
+            auto reader = make_reader(NYsonPull::NInput::FromMemory(text));
+            verify(reader, nrepeat);
+        }
+        {
+            TStringInput input(text);
+            auto reader = make_reader(NYsonPull::NInput::FromInputStream(&input, /* buffer_size = */ 1));
+            verify(reader, nrepeat);
+        }
+    }
+
+#ifdef _unix_
+    template <typename Here, typename There>
+    void pipe(Here&& reader, There&& writer) {
+        int fildes[2];
+        UNIT_ASSERT_VALUES_EQUAL_C(0, ::pipe(fildes), sys_error());
+        auto read_fd = fildes[0];
+        auto write_fd = fildes[1];
+
+        auto pid = ::fork();
+        UNIT_ASSERT_C(pid >= 0, sys_error());
+        if (pid > 0) {
+            // parent
+            UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(write_fd), sys_error());
+            reader(read_fd);
+            UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(read_fd), sys_error());
+        } else {
+            // child
+            UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(read_fd), sys_error());
+            UNIT_ASSERT_NO_EXCEPTION(writer(write_fd));
+            UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(write_fd), sys_error());
+            ::exit(0);
+        }
+        int stat_loc;
+        UNIT_ASSERT_VALUES_EQUAL_C(pid, ::waitpid(pid, &stat_loc, 0), sys_error());
+    }
+
+    template <typename Function>
+    void test_posix_fd(
+        Function make_writer,
+        size_t nrepeat,
+        size_t read_buffer_size,
+        size_t write_buffer_size) {
+        pipe(
+            [&](int fd) {
+                auto reader = make_reader(NYsonPull::NInput::FromPosixFd(fd, read_buffer_size));
+                verify(reader, nrepeat);
+            },
+            [&](int fd) {
+                auto writer = make_writer(NYsonPull::NOutput::FromPosixFd(fd, write_buffer_size));
+                generate(writer, nrepeat);
+            });
+    }
+
+    template <typename Function>
+    void test_stdio_file(
+        Function make_writer,
+        size_t nrepeat,
+        size_t read_buffer_size,
+        size_t write_buffer_size) {
+        pipe(
+            [&](int fd) {
+                auto file = ::fdopen(fd, "rb");
+                UNIT_ASSERT_C(file != nullptr, sys_error());
+                auto reader = make_reader(NYsonPull::NInput::FromStdioFile(file, read_buffer_size));
+                verify(reader, nrepeat);
+            },
+            [&](int fd) {
+                auto file = ::fdopen(fd, "wb");
+                Y_UNUSED(write_buffer_size);
+                auto writer = make_writer(NYsonPull::NOutput::FromStdioFile(file, write_buffer_size));
+                generate(writer, nrepeat);
+                fflush(file);
+            });
+    }
+#endif
+
+    NYsonPull::TWriter text(THolder<NYsonPull::NOutput::IStream> stream) {
+        return NYsonPull::MakeTextWriter(
+            std::move(stream),
+            NYsonPull::EStreamType::ListFragment);
+    }
+
+    NYsonPull::TWriter pretty_text(THolder<NYsonPull::NOutput::IStream> stream) {
+        return NYsonPull::MakePrettyTextWriter(
+            std::move(stream),
+            NYsonPull::EStreamType::ListFragment);
+    }
+
+    NYsonPull::TWriter binary(THolder<NYsonPull::NOutput::IStream> stream) {
+        return NYsonPull::MakeBinaryWriter(
+            std::move(stream),
+            NYsonPull::EStreamType::ListFragment);
+    }
+
+} // anonymous namespace
+
+Y_UNIT_TEST_SUITE(Loop) {
+    Y_UNIT_TEST(memory_pretty_text) {
+        test_memory(pretty_text, 100);
+    }
+
+    Y_UNIT_TEST(memory_text) {
+        test_memory(text, 100);
+    }
+
+    Y_UNIT_TEST(memory_binary) {
+        test_memory(binary, 100);
+    }
+
+#ifdef _unix_
+    Y_UNIT_TEST(posix_fd_pretty_text_buffered) {
+        test_posix_fd(pretty_text, 100, 1024, 1024);
+    }
+
+    Y_UNIT_TEST(posix_fd_pretty_text_unbuffered) {
+        test_posix_fd(pretty_text, 100, 1, 0);
+    }
+
+    Y_UNIT_TEST(posix_fd_text_buffered) {
+        test_posix_fd(text, 100, 1024, 1024);
+    }
+
+    Y_UNIT_TEST(posix_fd_text_unbuffered) {
+        test_posix_fd(text, 100, 1, 0);
+    }
+
+    Y_UNIT_TEST(posix_fd_binary_buffered) {
+        test_posix_fd(binary, 100, 1024, 1024);
+    }
+
+    Y_UNIT_TEST(posix_fd_binary_unbuffered) {
+        test_posix_fd(binary, 100, 1, 0);
+    }
+
+    Y_UNIT_TEST(stdio_file_pretty_text_buffered) {
+        test_stdio_file(pretty_text, 100, 1024, 1024);
+    }
+
+    Y_UNIT_TEST(stdio_file_pretty_text_unbuffered) {
+        test_stdio_file(pretty_text, 100, 1, 0);
+    }
+
+    Y_UNIT_TEST(stdio_file_text_buffered) {
+        test_stdio_file(text, 100, 1024, 1024);
+    }
+
+    Y_UNIT_TEST(stdio_file_text_unbuffered) {
+        test_stdio_file(text, 100, 1, 0);
+    }
+
+    Y_UNIT_TEST(stdio_file_binary_buffered) {
+        test_stdio_file(binary, 100, 1024, 1024);
+    }
+
+    Y_UNIT_TEST(stdio_file_binary_unbuffered) {
+        test_stdio_file(binary, 100, 1, 0);
+    }
+#endif
+} // Y_UNIT_TEST_SUITE(Loop)