diff options
author | max42 <max42@yandex-team.com> | 2023-06-30 03:37:03 +0300 |
---|---|---|
committer | max42 <max42@yandex-team.com> | 2023-06-30 03:37:03 +0300 |
commit | fac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a (patch) | |
tree | b8cbc1deb00309c7f1a7ab6df520a76cf0b5c6d7 /yt/cpp/mapreduce/common/helpers.cpp | |
parent | 7bf166b1a7ed0af927f230022b245af618e998c1 (diff) | |
download | ydb-fac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a.tar.gz |
YT-19324: move YT provider to ydb/library/yql
This commit is formed by the following script: https://paste.yandex-team.ru/6f92e4b8-efc5-4d34-948b-15ee2accd7e7/text.
This commit has zero effect on all projects that depend on YQL.
The summary of changes:
- `yql/providers/yt -> ydb/library/yql/providers/yt `- the whole implementation of YT provider is moved into YDB code base for further export as a part of YT YQL plugin shared library;
- `yql/providers/stat/{expr_nodes,uploader} -> ydb/library/yql/providers/stat/{expr_nodes,uploader}` - a small interface without implementation and the description of stat expr nodes;
- `yql/core/extract_predicate/ut -> ydb/library/yql/core/extract_predicate/ut`;
- `yql/core/{ut,ut_common} -> ydb/library/yql/core/{ut,ut_common}`;
- `yql/core` is gone;
- `yql/library/url_preprocessing -> ydb/library/yql/core/url_preprocessing`.
**NB**: all new targets inside `ydb/` are under `IF (NOT CMAKE_EXPORT)` clause which disables them from open-source cmake generation and ya make build. They will be enabled in the subsequent commits.
Diffstat (limited to 'yt/cpp/mapreduce/common/helpers.cpp')
-rw-r--r-- | yt/cpp/mapreduce/common/helpers.cpp | 126 |
1 files changed, 126 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/common/helpers.cpp b/yt/cpp/mapreduce/common/helpers.cpp new file mode 100644 index 00000000000..95924d812c9 --- /dev/null +++ b/yt/cpp/mapreduce/common/helpers.cpp @@ -0,0 +1,126 @@ +#include "helpers.h" + +#include <yt/cpp/mapreduce/interface/config.h> +#include <yt/cpp/mapreduce/interface/serialize.h> +#include <yt/cpp/mapreduce/interface/fluent.h> + +#include <library/cpp/yson/node/node_builder.h> +#include <library/cpp/yson/node/node_visitor.h> + +#include <library/cpp/yson/parser.h> +#include <library/cpp/yson/writer.h> + +#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_value.h> + +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/str.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TString NodeListToYsonString(const TNode::TListType& nodes) +{ + TStringStream stream; + ::NYson::TYsonWriter writer(&stream, NYson::EYsonFormat::Binary, ::NYson::EYsonType::ListFragment); + auto list = BuildYsonListFluently(&writer); + for (const auto& node : nodes) { + list.Item().Value(node); + } + return stream.Str(); +} + +TNode PathToNode(const TRichYPath& path) +{ + TNode result; + TNodeBuilder builder(&result); + Serialize(path, &builder); + return result; +} + +TNode PathToParamNode(const TRichYPath& path) +{ + return TNode()("path", PathToNode(path)); +} + +TString AttributesToYsonString(const TNode& node) +{ + return BuildYsonStringFluently().BeginMap() + .Item("attributes").Value(node) + .EndMap(); +} + +TString AttributeFilterToYsonString(const TAttributeFilter& filter) +{ + return BuildYsonStringFluently().BeginMap() + .Item("attributes").Value(filter) + .EndMap(); +} + +TNode NodeFromTableSchema(const TTableSchema& schema) +{ + TNode result; + TNodeBuilder builder(&result); + Serialize(schema, &builder); + return result; +} + +void MergeNodes(TNode& dst, const TNode& src) +{ + if (dst.IsMap() && src.IsMap()) { + auto& dstMap = dst.AsMap(); + const auto& srcMap = src.AsMap(); + for (const auto& srcItem : srcMap) { + const auto& key = srcItem.first; + auto dstItem = dstMap.find(key); + if (dstItem != dstMap.end()) { + MergeNodes(dstItem->second, srcItem.second); + } else { + dstMap[key] = srcItem.second; + } + } + } else { + if (dst.GetType() == src.GetType() && src.HasAttributes()) { + auto attributes = dst.GetAttributes(); + MergeNodes(attributes, src.GetAttributes()); + dst = src; + dst.Attributes() = attributes; + } else { + dst = src; + } + } +} + +TYPath AddPathPrefix(const TYPath& path, const TString& pathPrefix) +{ + if (path.StartsWith("//") || path.StartsWith("#")) { + return path; + } + return pathPrefix + path; +} + +TString GetWriteTableCommand(const TString& apiVersion) +{ + return apiVersion == "v2" ? "write" : "write_table"; +} + +TString GetReadTableCommand(const TString& apiVersion) +{ + return apiVersion == "v2" ? "read" : "read_table"; +} + +TString GetWriteFileCommand(const TString& apiVersion) +{ + return apiVersion == "v2" ? "upload" : "write_file"; +} + +TString GetReadFileCommand(const TString& apiVersion) +{ + return apiVersion == "v2" ? "download" : "read_file"; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT |