diff options
author | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
---|---|---|
committer | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
commit | 73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch) | |
tree | 188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/cpp/mapreduce/interface/format.cpp | |
parent | 528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff) | |
download | ydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz |
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build.
Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/cpp/mapreduce/interface/format.cpp')
-rw-r--r-- | yt/cpp/mapreduce/interface/format.cpp | 135 |
1 files changed, 135 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/interface/format.cpp b/yt/cpp/mapreduce/interface/format.cpp new file mode 100644 index 0000000000..f8318310a4 --- /dev/null +++ b/yt/cpp/mapreduce/interface/format.cpp @@ -0,0 +1,135 @@ +#include "format.h" +#include "protobuf_format.h" + +#include "errors.h" + +#include <google/protobuf/descriptor.h> +#include <google/protobuf/messagext.h> + +namespace NYT { + +TTableSchema CreateTableSchema( + const ::google::protobuf::Descriptor& messageDescriptor, + bool keepFieldsWithoutExtension) +{ + return NDetail::CreateTableSchemaImpl(messageDescriptor, keepFieldsWithoutExtension); +} + +//////////////////////////////////////////////////////////////////////////////// + +TFormat::TFormat(const TNode& config) + : Config(config) +{ } + + +TFormat TFormat::Protobuf( + const TVector<const ::google::protobuf::Descriptor*>& descriptors, + bool withDescriptors) +{ + if (withDescriptors) { + return TFormat(NDetail::MakeProtoFormatConfigWithDescriptors(descriptors)); + } else { + return TFormat(NDetail::MakeProtoFormatConfigWithTables(descriptors)); + } +} + +TFormat TFormat::YsonText() +{ + TNode config("yson"); + config.Attributes()("format", "text"); + return TFormat(config); +} + +TFormat TFormat::YsonBinary() +{ + TNode config("yson"); + config.Attributes()("format", "binary"); + return TFormat(config); +} + +TFormat TFormat::YaMRLenval() +{ + TNode config("yamr"); + config.Attributes()("lenval", true)("has_subkey", true); + return TFormat(config); +} + +TFormat TFormat::Json() +{ + return TFormat(TNode("json")); +} + +bool TFormat::IsTextYson() const +{ + if (!Config.IsString() || Config.AsString() != "yson") { + return false; + } + if (!Config.HasAttributes()) { + return false; + } + const auto& attributes = Config.GetAttributes(); + if (!attributes.HasKey("format") || attributes["format"] != TNode("text")) { + return false; + } + return true; +} + +bool TFormat::IsProtobuf() const +{ + return Config.IsString() && Config.AsString() == "protobuf"; +} + +bool TFormat::IsYamredDsv() const +{ + return Config.IsString() && Config.AsString() == "yamred_dsv"; +} + +static TString FormatName(const TFormat& format) +{ + if (!format.Config.IsString()) { + Y_VERIFY(format.Config.IsUndefined()); + return "<undefined>"; + } + return format.Config.AsString(); +} + +TYamredDsvAttributes TFormat::GetYamredDsvAttributes() const +{ + if (!IsYamredDsv()) { + ythrow TApiUsageError() << "Cannot get yamred_dsv attributes for " << FormatName(*this) << " format"; + } + TYamredDsvAttributes attributes; + + const auto& nodeAttributes = Config.GetAttributes(); + { + const auto& keyColumns = nodeAttributes["key_column_names"]; + if (!keyColumns.IsList()) { + ythrow yexception() << "Ill-formed format: key_column_names is of non-list type: " << keyColumns.GetType(); + } + for (auto& column : keyColumns.AsList()) { + if (!column.IsString()) { + ythrow yexception() << "Ill-formed format: key_column_names: " << column.GetType(); + } + attributes.KeyColumnNames.push_back(column.AsString()); + } + } + + if (nodeAttributes.HasKey("subkey_column_names")) { + const auto& subkeyColumns = nodeAttributes["subkey_column_names"]; + if (!subkeyColumns.IsList()) { + ythrow yexception() << "Ill-formed format: subkey_column_names is not a list: " << subkeyColumns.GetType(); + } + for (const auto& column : subkeyColumns.AsList()) { + if (!column.IsString()) { + ythrow yexception() << "Ill-formed format: non-string inside subkey_key_column_names: " << column.GetType(); + } + attributes.SubkeyColumnNames.push_back(column.AsString()); + } + } + + return attributes; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT |