aboutsummaryrefslogtreecommitdiffstats
path: root/yt/cpp/mapreduce/common/helpers.cpp
diff options
context:
space:
mode:
authormax42 <max42@yandex-team.com>2023-06-30 03:37:03 +0300
committermax42 <max42@yandex-team.com>2023-06-30 03:37:03 +0300
commitfac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a (patch)
treeb8cbc1deb00309c7f1a7ab6df520a76cf0b5c6d7 /yt/cpp/mapreduce/common/helpers.cpp
parent7bf166b1a7ed0af927f230022b245af618e998c1 (diff)
downloadydb-fac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a.tar.gz
YT-19324: move YT provider to ydb/library/yql
This commit is formed by the following script: https://paste.yandex-team.ru/6f92e4b8-efc5-4d34-948b-15ee2accd7e7/text. This commit has zero effect on all projects that depend on YQL. The summary of changes: - `yql/providers/yt -> ydb/library/yql/providers/yt `- the whole implementation of YT provider is moved into YDB code base for further export as a part of YT YQL plugin shared library; - `yql/providers/stat/{expr_nodes,uploader} -> ydb/library/yql/providers/stat/{expr_nodes,uploader}` - a small interface without implementation and the description of stat expr nodes; - `yql/core/extract_predicate/ut -> ydb/library/yql/core/extract_predicate/ut`; - `yql/core/{ut,ut_common} -> ydb/library/yql/core/{ut,ut_common}`; - `yql/core` is gone; - `yql/library/url_preprocessing -> ydb/library/yql/core/url_preprocessing`. **NB**: all new targets inside `ydb/` are under `IF (NOT CMAKE_EXPORT)` clause which disables them from open-source cmake generation and ya make build. They will be enabled in the subsequent commits.
Diffstat (limited to 'yt/cpp/mapreduce/common/helpers.cpp')
-rw-r--r--yt/cpp/mapreduce/common/helpers.cpp126
1 files changed, 126 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/common/helpers.cpp b/yt/cpp/mapreduce/common/helpers.cpp
new file mode 100644
index 00000000000..95924d812c9
--- /dev/null
+++ b/yt/cpp/mapreduce/common/helpers.cpp
@@ -0,0 +1,126 @@
+#include "helpers.h"
+
+#include <yt/cpp/mapreduce/interface/config.h>
+#include <yt/cpp/mapreduce/interface/serialize.h>
+#include <yt/cpp/mapreduce/interface/fluent.h>
+
+#include <library/cpp/yson/node/node_builder.h>
+#include <library/cpp/yson/node/node_visitor.h>
+
+#include <library/cpp/yson/parser.h>
+#include <library/cpp/yson/writer.h>
+
+#include <library/cpp/json/json_reader.h>
+#include <library/cpp/json/json_value.h>
+
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+#include <util/stream/str.h>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TString NodeListToYsonString(const TNode::TListType& nodes)
+{
+ TStringStream stream;
+ ::NYson::TYsonWriter writer(&stream, NYson::EYsonFormat::Binary, ::NYson::EYsonType::ListFragment);
+ auto list = BuildYsonListFluently(&writer);
+ for (const auto& node : nodes) {
+ list.Item().Value(node);
+ }
+ return stream.Str();
+}
+
+TNode PathToNode(const TRichYPath& path)
+{
+ TNode result;
+ TNodeBuilder builder(&result);
+ Serialize(path, &builder);
+ return result;
+}
+
+TNode PathToParamNode(const TRichYPath& path)
+{
+ return TNode()("path", PathToNode(path));
+}
+
+TString AttributesToYsonString(const TNode& node)
+{
+ return BuildYsonStringFluently().BeginMap()
+ .Item("attributes").Value(node)
+ .EndMap();
+}
+
+TString AttributeFilterToYsonString(const TAttributeFilter& filter)
+{
+ return BuildYsonStringFluently().BeginMap()
+ .Item("attributes").Value(filter)
+ .EndMap();
+}
+
+TNode NodeFromTableSchema(const TTableSchema& schema)
+{
+ TNode result;
+ TNodeBuilder builder(&result);
+ Serialize(schema, &builder);
+ return result;
+}
+
+void MergeNodes(TNode& dst, const TNode& src)
+{
+ if (dst.IsMap() && src.IsMap()) {
+ auto& dstMap = dst.AsMap();
+ const auto& srcMap = src.AsMap();
+ for (const auto& srcItem : srcMap) {
+ const auto& key = srcItem.first;
+ auto dstItem = dstMap.find(key);
+ if (dstItem != dstMap.end()) {
+ MergeNodes(dstItem->second, srcItem.second);
+ } else {
+ dstMap[key] = srcItem.second;
+ }
+ }
+ } else {
+ if (dst.GetType() == src.GetType() && src.HasAttributes()) {
+ auto attributes = dst.GetAttributes();
+ MergeNodes(attributes, src.GetAttributes());
+ dst = src;
+ dst.Attributes() = attributes;
+ } else {
+ dst = src;
+ }
+ }
+}
+
+TYPath AddPathPrefix(const TYPath& path, const TString& pathPrefix)
+{
+ if (path.StartsWith("//") || path.StartsWith("#")) {
+ return path;
+ }
+ return pathPrefix + path;
+}
+
+TString GetWriteTableCommand(const TString& apiVersion)
+{
+ return apiVersion == "v2" ? "write" : "write_table";
+}
+
+TString GetReadTableCommand(const TString& apiVersion)
+{
+ return apiVersion == "v2" ? "read" : "read_table";
+}
+
+TString GetWriteFileCommand(const TString& apiVersion)
+{
+ return apiVersion == "v2" ? "upload" : "write_file";
+}
+
+TString GetReadFileCommand(const TString& apiVersion)
+{
+ return apiVersion == "v2" ? "download" : "read_file";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT