aboutsummaryrefslogtreecommitdiffstats
path: root/yt/cpp/mapreduce/interface/protobuf_file_options_ut.cpp
diff options
context:
space:
mode:
authormax42 <max42@yandex-team.com>2023-06-30 03:37:03 +0300
committermax42 <max42@yandex-team.com>2023-06-30 03:37:03 +0300
commitfac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a (patch)
treeb8cbc1deb00309c7f1a7ab6df520a76cf0b5c6d7 /yt/cpp/mapreduce/interface/protobuf_file_options_ut.cpp
parent7bf166b1a7ed0af927f230022b245af618e998c1 (diff)
downloadydb-fac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a.tar.gz
YT-19324: move YT provider to ydb/library/yql
This commit is formed by the following script: https://paste.yandex-team.ru/6f92e4b8-efc5-4d34-948b-15ee2accd7e7/text. This commit has zero effect on all projects that depend on YQL. The summary of changes: - `yql/providers/yt -> ydb/library/yql/providers/yt `- the whole implementation of YT provider is moved into YDB code base for further export as a part of YT YQL plugin shared library; - `yql/providers/stat/{expr_nodes,uploader} -> ydb/library/yql/providers/stat/{expr_nodes,uploader}` - a small interface without implementation and the description of stat expr nodes; - `yql/core/extract_predicate/ut -> ydb/library/yql/core/extract_predicate/ut`; - `yql/core/{ut,ut_common} -> ydb/library/yql/core/{ut,ut_common}`; - `yql/core` is gone; - `yql/library/url_preprocessing -> ydb/library/yql/core/url_preprocessing`. **NB**: all new targets inside `ydb/` are under `IF (NOT CMAKE_EXPORT)` clause which disables them from open-source cmake generation and ya make build. They will be enabled in the subsequent commits.
Diffstat (limited to 'yt/cpp/mapreduce/interface/protobuf_file_options_ut.cpp')
-rw-r--r--yt/cpp/mapreduce/interface/protobuf_file_options_ut.cpp271
1 files changed, 271 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/interface/protobuf_file_options_ut.cpp b/yt/cpp/mapreduce/interface/protobuf_file_options_ut.cpp
new file mode 100644
index 0000000000..5ffa9564d7
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/protobuf_file_options_ut.cpp
@@ -0,0 +1,271 @@
+#include "errors.h"
+#include "format.h"
+#include "common_ut.h"
+
+#include <yt/cpp/mapreduce/interface/protobuf_file_options_ut.pb.h>
+
+#include <yt/cpp/mapreduce/tests/yt_unittest_lib/yt_unittest_lib.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NYT;
+
+Y_UNIT_TEST_SUITE(ProtobufFileOptions)
+{
+ NTi::TTypePtr GetUrlRowType(bool required)
+ {
+ static const NTi::TTypePtr structType = NTi::Struct({
+ {"Host", ToTypeV3(EValueType::VT_STRING, false)},
+ {"Path", ToTypeV3(EValueType::VT_STRING, false)},
+ {"HttpCode", ToTypeV3(EValueType::VT_INT32, false)}});
+ return required ? structType : NTi::TTypePtr(NTi::Optional(structType));
+ }
+
+ Y_UNIT_TEST(TRowFieldSerializationOption)
+ {
+ const auto schema = CreateTableSchema<NTestingFileOptions::TRowFieldSerializationOption>();
+
+ ASSERT_SERIALIZABLES_EQUAL(schema, TTableSchema()
+ .AddColumn(TColumnSchema().Name("UrlRow_1").Type(ToTypeV3(EValueType::VT_STRING, false)))
+ .AddColumn(TColumnSchema().Name("UrlRow_2").Type(GetUrlRowType(false))));
+ }
+
+ Y_UNIT_TEST(TRowMixedSerializationOptions)
+ {
+ const auto schema = CreateTableSchema<NTestingFileOptions::TRowMixedSerializationOptions>();
+
+ ASSERT_SERIALIZABLES_EQUAL(schema, TTableSchema()
+ .AddColumn(TColumnSchema().Name("UrlRow_1").Type(ToTypeV3(EValueType::VT_STRING, false)))
+ .AddColumn(TColumnSchema().Name("UrlRow_2").Type(GetUrlRowType(false))));
+ }
+
+ Y_UNIT_TEST(FieldSortOrder)
+ {
+ const auto schema = CreateTableSchema<NTestingFileOptions::TFieldSortOrder>();
+
+ auto asInProtoFile = NTi::Optional(NTi::Struct({
+ {"x", NTi::Optional(NTi::Int64())},
+ {"y", NTi::Optional(NTi::String())},
+ {"z", NTi::Optional(NTi::Bool())},
+ }));
+ auto byFieldNumber = NTi::Optional(NTi::Struct({
+ {"z", NTi::Optional(NTi::Bool())},
+ {"x", NTi::Optional(NTi::Int64())},
+ {"y", NTi::Optional(NTi::String())},
+ }));
+
+ ASSERT_SERIALIZABLES_EQUAL(schema, TTableSchema()
+ .AddColumn(TColumnSchema().Name("EmbeddedDefault").Type(asInProtoFile))
+ .AddColumn(TColumnSchema().Name("EmbeddedAsInProtoFile").Type(asInProtoFile))
+ .AddColumn(TColumnSchema().Name("EmbeddedByFieldNumber").Type(byFieldNumber)));
+ }
+
+ Y_UNIT_TEST(Map)
+ {
+ const auto schema = CreateTableSchema<NTestingFileOptions::TWithMap>();
+
+ auto createKeyValueStruct = [] (NTi::TTypePtr key, NTi::TTypePtr value) {
+ return NTi::List(NTi::Struct({
+ {"key", NTi::Optional(key)},
+ {"value", NTi::Optional(value)},
+ }));
+ };
+
+ auto embedded = NTi::Struct({
+ {"x", NTi::Optional(NTi::Int64())},
+ {"y", NTi::Optional(NTi::String())},
+ });
+
+ ASSERT_SERIALIZABLES_EQUAL(schema, TTableSchema()
+ .AddColumn(TColumnSchema()
+ .Name("MapDefault")
+ .Type(createKeyValueStruct(NTi::Int64(), embedded)))
+ .AddColumn(TColumnSchema()
+ .Name("MapDict")
+ .Type(NTi::Dict(NTi::Int64(), embedded))));
+ }
+
+ Y_UNIT_TEST(Oneof)
+ {
+ const auto schema = CreateTableSchema<NTestingFileOptions::TWithOneof>();
+
+ auto embedded = NTi::Struct({
+ {"x", NTi::Optional(NTi::Int64())},
+ {"y", NTi::Optional(NTi::String())},
+ });
+
+ auto defaultVariantType = NTi::Optional(NTi::Struct({
+ {"field", NTi::Optional(NTi::String())},
+ {"Oneof2", NTi::Optional(NTi::Variant(NTi::Struct({
+ {"y2", NTi::String()},
+ {"z2", embedded},
+ {"x2", NTi::Int64()},
+ })))},
+ {"x1", NTi::Optional(NTi::Int64())},
+ {"y1", NTi::Optional(NTi::String())},
+ {"z1", NTi::Optional(embedded)},
+ }));
+
+ auto noDefaultType = NTi::Optional(NTi::Struct({
+ {"field", NTi::Optional(NTi::String())},
+ {"y2", NTi::Optional(NTi::String())},
+ {"z2", NTi::Optional(embedded)},
+ {"x2", NTi::Optional(NTi::Int64())},
+ {"x1", NTi::Optional(NTi::Int64())},
+ {"y1", NTi::Optional(NTi::String())},
+ {"z1", NTi::Optional(embedded)},
+ }));
+
+ ASSERT_SERIALIZABLES_EQUAL(schema, TTableSchema()
+ .AddColumn(TColumnSchema()
+ .Name("DefaultVariant")
+ .Type(defaultVariantType)
+ )
+ .AddColumn(TColumnSchema()
+ .Name("NoDefault")
+ .Type(noDefaultType)
+ )
+ .AddColumn(TColumnSchema()
+ .Name("SerializationProtobuf")
+ .Type(NTi::Optional(NTi::Struct({
+ {"x1", NTi::Optional(NTi::Int64())},
+ {"y1", NTi::Optional(NTi::String())},
+ {"z1", NTi::Optional(NTi::String())},
+ })))
+ )
+ .AddColumn(TColumnSchema()
+ .Name("MemberOfTopLevelOneof")
+ .Type(NTi::Optional(NTi::Int64()))
+ )
+ );
+ }
+}
+
+static TNode GetColumns(const TFormat& format, int tableIndex = 0)
+{
+ return format.Config.GetAttributes()["tables"][tableIndex]["columns"];
+}
+
+Y_UNIT_TEST_SUITE(ProtobufFormatFileOptions)
+{
+ Y_UNIT_TEST(TRowFieldSerializationOption)
+ {
+ const auto format = TFormat::Protobuf<NTestingFileOptions::TRowFieldSerializationOption>();
+ auto columns = GetColumns(format);
+
+ UNIT_ASSERT_VALUES_EQUAL(columns[0]["name"], "UrlRow_1");
+ UNIT_ASSERT_VALUES_EQUAL(columns[0]["proto_type"], "message");
+ UNIT_ASSERT_VALUES_EQUAL(columns[0]["field_number"], 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(columns[1]["name"], "UrlRow_2");
+ UNIT_ASSERT_VALUES_EQUAL(columns[1]["proto_type"], "structured_message");
+ UNIT_ASSERT_VALUES_EQUAL(columns[1]["field_number"], 2);
+ const auto& fields = columns[1]["fields"];
+ UNIT_ASSERT_VALUES_EQUAL(fields[0]["name"], "Host");
+ UNIT_ASSERT_VALUES_EQUAL(fields[0]["proto_type"], "string");
+ UNIT_ASSERT_VALUES_EQUAL(fields[0]["field_number"], 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(fields[1]["name"], "Path");
+ UNIT_ASSERT_VALUES_EQUAL(fields[1]["proto_type"], "string");
+ UNIT_ASSERT_VALUES_EQUAL(fields[1]["field_number"], 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(fields[2]["name"], "HttpCode");
+ UNIT_ASSERT_VALUES_EQUAL(fields[2]["proto_type"], "sint32");
+ UNIT_ASSERT_VALUES_EQUAL(fields[2]["field_number"], 3);
+ }
+
+ Y_UNIT_TEST(Map)
+ {
+ const auto format = TFormat::Protobuf<NTestingFileOptions::TWithMap>();
+ auto columns = GetColumns(format);
+
+ UNIT_ASSERT_VALUES_EQUAL(columns.Size(), 2);
+ {
+ const auto& column = columns[0];
+ UNIT_ASSERT_VALUES_EQUAL(column["name"], "MapDefault");
+ UNIT_ASSERT_VALUES_EQUAL(column["proto_type"], "structured_message");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"].Size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][0]["proto_type"], "int64");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][1]["proto_type"], "structured_message");
+ }
+ {
+ const auto& column = columns[1];
+ UNIT_ASSERT_VALUES_EQUAL(column["name"], "MapDict");
+ UNIT_ASSERT_VALUES_EQUAL(column["proto_type"], "structured_message");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"].Size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][0]["proto_type"], "int64");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][1]["proto_type"], "structured_message");
+ }
+ }
+
+ Y_UNIT_TEST(Oneof)
+ {
+ const auto format = TFormat::Protobuf<NTestingFileOptions::TWithOneof>();
+ auto columns = GetColumns(format);
+
+ UNIT_ASSERT_VALUES_EQUAL(columns.Size(), 4);
+
+ {
+ const auto& column = columns[0];
+ UNIT_ASSERT_VALUES_EQUAL(column["name"], "DefaultVariant");
+ UNIT_ASSERT_VALUES_EQUAL(column["proto_type"], "structured_message");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"].Size(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][0]["name"], "field");
+
+ const auto& oneof2 = column["fields"][1];
+ UNIT_ASSERT_VALUES_EQUAL(oneof2["name"], "Oneof2");
+ UNIT_ASSERT_VALUES_EQUAL(oneof2["proto_type"], "oneof");
+ UNIT_ASSERT_VALUES_EQUAL(oneof2["fields"][0]["name"], "y2");
+ UNIT_ASSERT_VALUES_EQUAL(oneof2["fields"][1]["name"], "z2");
+ UNIT_ASSERT_VALUES_EQUAL(oneof2["fields"][1]["proto_type"], "structured_message");
+ const auto& embeddedFields = oneof2["fields"][1]["fields"];
+ UNIT_ASSERT_VALUES_EQUAL(embeddedFields[0]["name"], "x");
+ UNIT_ASSERT_VALUES_EQUAL(embeddedFields[1]["name"], "y");
+
+ UNIT_ASSERT_VALUES_EQUAL(oneof2["fields"][2]["name"], "x2");
+
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][2]["name"], "x1");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][3]["name"], "y1");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][4]["name"], "z1");
+ };
+
+ {
+ const auto& column = columns[1];
+ UNIT_ASSERT_VALUES_EQUAL(column["name"], "NoDefault");
+ UNIT_ASSERT_VALUES_EQUAL(column["proto_type"], "structured_message");
+ const auto& fields = column["fields"];
+ UNIT_ASSERT_VALUES_EQUAL(fields.Size(), 7);
+
+ UNIT_ASSERT_VALUES_EQUAL(fields[0]["name"], "field");
+
+ UNIT_ASSERT_VALUES_EQUAL(fields[1]["name"], "y2");
+
+ UNIT_ASSERT_VALUES_EQUAL(fields[2]["name"], "z2");
+ UNIT_ASSERT_VALUES_EQUAL(fields[2]["proto_type"], "structured_message");
+ const auto& embeddedFields = fields[2]["fields"];
+ UNIT_ASSERT_VALUES_EQUAL(embeddedFields[0]["name"], "x");
+ UNIT_ASSERT_VALUES_EQUAL(embeddedFields[1]["name"], "y");
+
+ UNIT_ASSERT_VALUES_EQUAL(fields[3]["name"], "x2");
+
+ UNIT_ASSERT_VALUES_EQUAL(fields[4]["name"], "x1");
+ UNIT_ASSERT_VALUES_EQUAL(fields[5]["name"], "y1");
+ UNIT_ASSERT_VALUES_EQUAL(fields[6]["name"], "z1");
+ };
+
+ {
+ const auto& column = columns[2];
+ UNIT_ASSERT_VALUES_EQUAL(column["name"], "SerializationProtobuf");
+ UNIT_ASSERT_VALUES_EQUAL(column["proto_type"], "structured_message");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"].Size(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][0]["name"], "x1");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][1]["name"], "y1");
+ UNIT_ASSERT_VALUES_EQUAL(column["fields"][2]["name"], "z1");
+ }
+ {
+ const auto& column = columns[3];
+ UNIT_ASSERT_VALUES_EQUAL(column["name"], "MemberOfTopLevelOneof");
+ UNIT_ASSERT_VALUES_EQUAL(column["proto_type"], "int64");
+ }
+ }
+}