diff options
| author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
|---|---|---|
| committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
| commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
| tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/yson_pull/ut/reader_ut.cpp | |
| download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz | |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/yson_pull/ut/reader_ut.cpp')
| -rw-r--r-- | library/cpp/yson_pull/ut/reader_ut.cpp | 410 |
1 files changed, 410 insertions, 0 deletions
diff --git a/library/cpp/yson_pull/ut/reader_ut.cpp b/library/cpp/yson_pull/ut/reader_ut.cpp new file mode 100644 index 0000000000..1184265ddb --- /dev/null +++ b/library/cpp/yson_pull/ut/reader_ut.cpp @@ -0,0 +1,410 @@ +#include <library/cpp/yson_pull/exceptions.h> +#include <library/cpp/yson_pull/range.h> +#include <library/cpp/yson_pull/reader.h> +#include <library/cpp/yson_pull/detail/cescape.h> +#include <library/cpp/yson_pull/detail/macros.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace { + NYsonPull::TReader memory_reader(TStringBuf data, NYsonPull::EStreamType mode) { + return NYsonPull::TReader( + NYsonPull::NInput::FromMemory(data), + mode); + } + + template <typename T> + void expect_scalar(const NYsonPull::TScalar& scalar, T value) { + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{value}, scalar); + } + + template <> + void expect_scalar(const NYsonPull::TScalar& scalar, double value) { + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::Float64, scalar.Type()); + + auto scalarValue = scalar.AsFloat64(); + auto message = TStringBuilder() << "expected " << value << ", got " << scalarValue; + + if (std::isfinite(value)) { + UNIT_ASSERT_C(std::isfinite(scalarValue), message); + UNIT_ASSERT_DOUBLES_EQUAL(value, scalarValue, 1e-5); + } else if (std::isnan(value)) { + UNIT_ASSERT_C(std::isnan(scalarValue), message); + } else if (value > 0) { + UNIT_ASSERT_C(std::isinf(scalarValue) && (scalarValue > 0), message); + } else { + UNIT_ASSERT_C(std::isinf(scalarValue) && (scalarValue < 0), message); + } + } + + template <typename T> + void test_scalar(TStringBuf data, T value) { + // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); + auto reader = memory_reader(data, NYsonPull::EStreamType::Node); + + try { + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); + { + auto& event = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, event.Type()); + expect_scalar(event.AsScalar(), value); + } + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); + } catch (const std::exception& err) { + UNIT_FAIL(err.what()); + } + } + + void consume(TStringBuf data, NYsonPull::EStreamType mode = NYsonPull::EStreamType::Node) { + // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); + auto input_range = NYsonPull::TStreamEventsRange( + NYsonPull::NInput::FromMemory(data), + mode); + for (auto& event : input_range) { + Y_UNUSED(event); + } + } + +#define ACCEPT(data) UNIT_ASSERT_NO_EXCEPTION(consume(data)) +#define REJECT(data) UNIT_ASSERT_EXCEPTION(consume(data), NYsonPull::NException::TBadInput) + +#define ACCEPT2(data, mode) UNIT_ASSERT_NO_EXCEPTION(consume(data, mode)) +#define REJECT2(data, mode) UNIT_ASSERT_EXCEPTION(consume(data, mode), NYsonPull::NException::TBadInput) + +} // anonymous namespace + +Y_UNIT_TEST_SUITE(Reader) { + Y_UNIT_TEST(ScalarEntity) { + test_scalar(TStringBuf("#"), NYsonPull::TScalar{}); + } + + Y_UNIT_TEST(ScalarBoolean) { + test_scalar(TStringBuf("%true"), true); + test_scalar(TStringBuf("%false"), false); + + test_scalar(TStringBuf("\x05"sv), true); + test_scalar(TStringBuf("\x04"sv), false); + + REJECT("%"); + REJECT("%trueth"); + REJECT("%tru"); + REJECT("%falseth"); + REJECT("%fals"); + REJECT("%hithere"); + } + + Y_UNIT_TEST(ScalarInt64) { + test_scalar(TStringBuf("1"), i64{1}); + test_scalar(TStringBuf("+1"), i64{1}); + test_scalar(TStringBuf("100000"), i64{100000}); + test_scalar(TStringBuf("+100000"), i64{100000}); + test_scalar(TStringBuf("-100000"), i64{-100000}); + test_scalar(TStringBuf("9223372036854775807"), i64{9223372036854775807}); + test_scalar(TStringBuf("+9223372036854775807"), i64{9223372036854775807}); + + test_scalar(TStringBuf("\x02\x02"sv), i64{1}); + test_scalar(TStringBuf("\x02\xc0\x9a\x0c"sv), i64{100000}); + test_scalar(TStringBuf("\x02\xbf\x9a\x0c"sv), i64{-100000}); + test_scalar(TStringBuf("\x02\xfe\xff\xff\xff\xff\xff\xff\xff\xff\x01"sv), i64{9223372036854775807}); + + REJECT("1a2"); + REJECT("1-1-1-1"); + REJECT("1+0"); + } + + Y_UNIT_TEST(SclarUInt64) { + test_scalar(TStringBuf("1u"), ui64{1}); + test_scalar(TStringBuf("+1u"), ui64{1}); + test_scalar(TStringBuf("100000u"), ui64{100000}); + test_scalar(TStringBuf("+100000u"), ui64{100000}); + test_scalar(TStringBuf("9223372036854775807u"), ui64{9223372036854775807u}); + test_scalar(TStringBuf("+9223372036854775807u"), ui64{9223372036854775807u}); + test_scalar(TStringBuf("18446744073709551615u"), ui64{18446744073709551615u}); + test_scalar(TStringBuf("+18446744073709551615u"), ui64{18446744073709551615u}); + + REJECT("1a2u"); + REJECT("1-1-1-1u"); + REJECT("1+0u"); + + // TODO: binary + } + + Y_UNIT_TEST(ScalarFloat64) { + test_scalar(TStringBuf("0.0"), double{0.0}); + test_scalar(TStringBuf("+0.0"), double{0.0}); + test_scalar(TStringBuf("+.0"), double{0.0}); + test_scalar(TStringBuf("+.5"), double{0.5}); + test_scalar(TStringBuf("-.5"), double{-0.5}); + test_scalar(TStringBuf("1.0"), double{1.0}); + test_scalar(TStringBuf("+1.0"), double{1.0}); + test_scalar(TStringBuf("-1.0"), double{-1.0}); + test_scalar(TStringBuf("1000.0"), double{1000.0}); + test_scalar(TStringBuf("+1000.0"), double{1000.0}); + test_scalar(TStringBuf("-1000.0"), double{-1000.0}); + test_scalar(TStringBuf("1e12"), double{1e12}); + test_scalar(TStringBuf("1e+12"), double{1e12}); + test_scalar(TStringBuf("+1e+12"), double{1e12}); + test_scalar(TStringBuf("-1e+12"), double{-1e12}); + test_scalar(TStringBuf("1e-12"), double{1e-12}); + test_scalar(TStringBuf("+1e-12"), double{1e-12}); + test_scalar(TStringBuf("-1e-12"), double{-1e-12}); + + test_scalar(TStringBuf("\x03\x00\x00\x00\x00\x00\x00\x00\x00"sv), double{0.0}); + + test_scalar( + TStringBuf("\x03\x00\x00\x00\x00\x00\x00\xf8\x7f"sv), + double{std::numeric_limits<double>::quiet_NaN()}); + test_scalar( + TStringBuf("\x03\x00\x00\x00\x00\x00\x00\xf0\x7f"sv), + double{std::numeric_limits<double>::infinity()}); + test_scalar( + TStringBuf("\x03\x00\x00\x00\x00\x00\x00\xf0\xff"sv), + double{-std::numeric_limits<double>::infinity()}); + + test_scalar( + TStringBuf("%nan"), + double{std::numeric_limits<double>::quiet_NaN()}); + test_scalar( + TStringBuf("%inf"), + double{std::numeric_limits<double>::infinity()}); + test_scalar( + TStringBuf("%-inf"), + double{-std::numeric_limits<double>::infinity()}); + + REJECT("++0.0"); + REJECT("++1.0"); + REJECT("++.1"); + REJECT("1.0.0"); + //REJECT("1e+10000"); + REJECT(TStringBuf("\x03\x00\x00\x00\x00\x00\x00\x00"sv)); + + // XXX: Questionable behaviour? + ACCEPT("+.0"); + ACCEPT("-.0"); + // XXX: Rejected on Mac OS, accepted on Linux (?!) + //REJECT(".0"); + //REJECT(".5"); + + REJECT("%NaN"); + REJECT("%+inf"); + REJECT("%infinity"); + REJECT("%na"); + REJECT("%in"); + REJECT("%-in"); + } + + Y_UNIT_TEST(ScalarString) { + test_scalar(TStringBuf(R"(foobar)"), TStringBuf("foobar")); + test_scalar(TStringBuf(R"(foobar11)"), TStringBuf("foobar11")); + test_scalar(TStringBuf(R"("foobar")"), TStringBuf("foobar")); + // wat? "\x0cf" parsed as a single char? no way! + test_scalar("\x01\x0c" "foobar"sv, + TStringBuf("foobar")); + + REJECT(R"("foobar)"); + REJECT("\x01\x0c" "fooba"sv); + REJECT("\x01\x0d" "foobar"sv); // negative length + } + + Y_UNIT_TEST(EmptyList) { + auto reader = memory_reader("[]", NYsonPull::EStreamType::Node); + + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginList, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndList, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); + + REJECT("["); + REJECT("]"); + } + + Y_UNIT_TEST(EmptyMap) { + auto reader = memory_reader("{}", NYsonPull::EStreamType::Node); + + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginMap, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndMap, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); + + REJECT("{"); + REJECT("}"); + } + + Y_UNIT_TEST(Sample) { + auto reader = memory_reader( + R"({"11"=11;"nothing"=#;"zero"=0.;"foo"="bar";"list"=[1;2;3]})", + NYsonPull::EStreamType::Node); + + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginMap, reader.NextEvent().Type()); + + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("11"), e.AsString()); + } + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{11}}, e.AsScalar()); + } + + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("nothing"), e.AsString()); + } + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{}, e.AsScalar()); + } + + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("zero"), e.AsString()); + } + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{0.0}, e.AsScalar()); + } + + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("foo"), e.AsString()); + } + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{TStringBuf("bar")}, e.AsScalar()); + } + + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf("list"), e.AsString()); + } + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginList, reader.NextEvent().Type()); + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{1}}, e.AsScalar()); + } + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{2}}, e.AsScalar()); + } + { + auto& e = reader.NextEvent(); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{3}}, e.AsScalar()); + } + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndList, reader.NextEvent().Type()); + + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndMap, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); + } + + Y_UNIT_TEST(Accept) { + ACCEPT("[]"); + ACCEPT("{}"); + ACCEPT("<>[]"); + ACCEPT("<>{}"); + ACCEPT("[{};{};{}]"); + ACCEPT("[{};{};{};]"); + ACCEPT("[<>{};<>{};<>{}]"); + ACCEPT("[<>{};<>{};<>{};]"); + + ACCEPT("foo"); + ACCEPT("[foo]"); + ACCEPT("[foo;]"); + ACCEPT("{foo=foo}"); + ACCEPT("{foo=foo;}"); + ACCEPT("<>{foo=foo}"); + ACCEPT("{foo=<foo=foo>foo}"); + ACCEPT("{foo=<foo=foo;>foo}"); + ACCEPT("{foo=<foo=foo>[foo;foo]}"); + } + + Y_UNIT_TEST(Reject) { + REJECT("["); + REJECT("{"); + REJECT("<"); + + REJECT("[[}]"); + REJECT("<>{]"); + REJECT("[>]"); + + REJECT("<><>[]"); + REJECT("[<>;<>]"); + + REJECT("{<>foo=foo}"); + REJECT("{foo=<>}"); + REJECT("{foo}"); + + REJECT("<a=b>"); + REJECT("<>"); + + REJECT("@"); + } + + Y_UNIT_TEST(ReadPastEnd) { + auto reader = memory_reader("#", NYsonPull::EStreamType::Node); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, reader.NextEvent().Type()); + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); + UNIT_ASSERT_EXCEPTION(reader.NextEvent(), NYsonPull::NException::TBadInput); + } + + Y_UNIT_TEST(BadInput) { + // max_size<ui32> < varint size < max_size<ui64> + auto t = TString("\x01\xff\xff\xff\xff\xff\xff\xff\xff"); + auto reader = memory_reader(t, NYsonPull::EStreamType::Node); + + UNIT_ASSERT_EQUAL(reader.NextEvent().Type(), NYsonPull::EEventType::BeginStream); + UNIT_ASSERT_EXCEPTION(reader.NextEvent(), NYsonPull::NException::TBadInput); + } + + Y_UNIT_TEST(StreamType) { + REJECT2("", NYsonPull::EStreamType::Node); + ACCEPT2("", NYsonPull::EStreamType::ListFragment); + ACCEPT2("", NYsonPull::EStreamType::MapFragment); + + ACCEPT2("[1]", NYsonPull::EStreamType::Node); + ACCEPT2("[1]", NYsonPull::EStreamType::ListFragment); + REJECT2("[1]", NYsonPull::EStreamType::MapFragment); + + ACCEPT2("<foo=bar>[1]", NYsonPull::EStreamType::Node); + ACCEPT2("<foo=bar>[1]", NYsonPull::EStreamType::ListFragment); + REJECT2("<foo=bar>[1]", NYsonPull::EStreamType::MapFragment); + + ACCEPT2(" [1] \t \t ", NYsonPull::EStreamType::Node); + ACCEPT2(" [1] \t \t ", NYsonPull::EStreamType::ListFragment); + REJECT2(" [1] \t \t ", NYsonPull::EStreamType::MapFragment); + + REJECT2("[1];", NYsonPull::EStreamType::Node); + ACCEPT2("[1];", NYsonPull::EStreamType::ListFragment); + REJECT2("[1];", NYsonPull::EStreamType::MapFragment); + + REJECT2("[1]; foobar", NYsonPull::EStreamType::Node); + ACCEPT2("[1]; foobar", NYsonPull::EStreamType::ListFragment); + REJECT2("[1]; foobar", NYsonPull::EStreamType::MapFragment); + + REJECT2("a=[1]", NYsonPull::EStreamType::Node); + REJECT2("a=[1]", NYsonPull::EStreamType::ListFragment); + ACCEPT2("a=[1]", NYsonPull::EStreamType::MapFragment); + + REJECT2("a=[1]; ", NYsonPull::EStreamType::Node); + REJECT2("a=[1]; ", NYsonPull::EStreamType::ListFragment); + ACCEPT2("a=[1]; ", NYsonPull::EStreamType::MapFragment); + + REJECT2("a=[1]; b=foobar", NYsonPull::EStreamType::Node); + REJECT2("a=[1]; b=foobar", NYsonPull::EStreamType::ListFragment); + ACCEPT2("a=[1]; b=foobar", NYsonPull::EStreamType::MapFragment); + } + +} // Y_UNIT_TEST_SUITE(Reader) |