aboutsummaryrefslogtreecommitdiffstats
path: root/yt/cpp/mapreduce/interface/format.cpp
diff options
context:
space:
mode:
authormax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
committermax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
commit73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch)
tree188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/cpp/mapreduce/interface/format.cpp
parent528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff)
downloadydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build. Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/cpp/mapreduce/interface/format.cpp')
-rw-r--r--yt/cpp/mapreduce/interface/format.cpp135
1 files changed, 135 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/interface/format.cpp b/yt/cpp/mapreduce/interface/format.cpp
new file mode 100644
index 0000000000..f8318310a4
--- /dev/null
+++ b/yt/cpp/mapreduce/interface/format.cpp
@@ -0,0 +1,135 @@
+#include "format.h"
+#include "protobuf_format.h"
+
+#include "errors.h"
+
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/messagext.h>
+
+namespace NYT {
+
+TTableSchema CreateTableSchema(
+ const ::google::protobuf::Descriptor& messageDescriptor,
+ bool keepFieldsWithoutExtension)
+{
+ return NDetail::CreateTableSchemaImpl(messageDescriptor, keepFieldsWithoutExtension);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+TFormat::TFormat(const TNode& config)
+ : Config(config)
+{ }
+
+
+TFormat TFormat::Protobuf(
+ const TVector<const ::google::protobuf::Descriptor*>& descriptors,
+ bool withDescriptors)
+{
+ if (withDescriptors) {
+ return TFormat(NDetail::MakeProtoFormatConfigWithDescriptors(descriptors));
+ } else {
+ return TFormat(NDetail::MakeProtoFormatConfigWithTables(descriptors));
+ }
+}
+
+TFormat TFormat::YsonText()
+{
+ TNode config("yson");
+ config.Attributes()("format", "text");
+ return TFormat(config);
+}
+
+TFormat TFormat::YsonBinary()
+{
+ TNode config("yson");
+ config.Attributes()("format", "binary");
+ return TFormat(config);
+}
+
+TFormat TFormat::YaMRLenval()
+{
+ TNode config("yamr");
+ config.Attributes()("lenval", true)("has_subkey", true);
+ return TFormat(config);
+}
+
+TFormat TFormat::Json()
+{
+ return TFormat(TNode("json"));
+}
+
+bool TFormat::IsTextYson() const
+{
+ if (!Config.IsString() || Config.AsString() != "yson") {
+ return false;
+ }
+ if (!Config.HasAttributes()) {
+ return false;
+ }
+ const auto& attributes = Config.GetAttributes();
+ if (!attributes.HasKey("format") || attributes["format"] != TNode("text")) {
+ return false;
+ }
+ return true;
+}
+
+bool TFormat::IsProtobuf() const
+{
+ return Config.IsString() && Config.AsString() == "protobuf";
+}
+
+bool TFormat::IsYamredDsv() const
+{
+ return Config.IsString() && Config.AsString() == "yamred_dsv";
+}
+
+static TString FormatName(const TFormat& format)
+{
+ if (!format.Config.IsString()) {
+ Y_VERIFY(format.Config.IsUndefined());
+ return "<undefined>";
+ }
+ return format.Config.AsString();
+}
+
+TYamredDsvAttributes TFormat::GetYamredDsvAttributes() const
+{
+ if (!IsYamredDsv()) {
+ ythrow TApiUsageError() << "Cannot get yamred_dsv attributes for " << FormatName(*this) << " format";
+ }
+ TYamredDsvAttributes attributes;
+
+ const auto& nodeAttributes = Config.GetAttributes();
+ {
+ const auto& keyColumns = nodeAttributes["key_column_names"];
+ if (!keyColumns.IsList()) {
+ ythrow yexception() << "Ill-formed format: key_column_names is of non-list type: " << keyColumns.GetType();
+ }
+ for (auto& column : keyColumns.AsList()) {
+ if (!column.IsString()) {
+ ythrow yexception() << "Ill-formed format: key_column_names: " << column.GetType();
+ }
+ attributes.KeyColumnNames.push_back(column.AsString());
+ }
+ }
+
+ if (nodeAttributes.HasKey("subkey_column_names")) {
+ const auto& subkeyColumns = nodeAttributes["subkey_column_names"];
+ if (!subkeyColumns.IsList()) {
+ ythrow yexception() << "Ill-formed format: subkey_column_names is not a list: " << subkeyColumns.GetType();
+ }
+ for (const auto& column : subkeyColumns.AsList()) {
+ if (!column.IsString()) {
+ ythrow yexception() << "Ill-formed format: non-string inside subkey_key_column_names: " << column.GetType();
+ }
+ attributes.SubkeyColumnNames.push_back(column.AsString());
+ }
+ }
+
+ return attributes;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT