summaryrefslogtreecommitdiffstats
path: root/yt/cpp/mapreduce/client/py_helpers.cpp
diff options
context:
space:
mode:
authormax42 <[email protected]>2023-06-30 11:13:34 +0300
committermax42 <[email protected]>2023-06-30 11:13:34 +0300
commit3e1899838408bbad47622007aa382bc8a2b01f87 (patch)
tree0f21c1e6add187ddb6c3ccc048a7d640ce03fb87 /yt/cpp/mapreduce/client/py_helpers.cpp
parent5463eb3f5e72a86f858a3d27c886470a724ede34 (diff)
Revert "YT-19324: move YT provider to ydb/library/yql"
This reverts commit ca272f12fdd0e8d5c3e957fc87939148f1caaf72, reversing changes made to 49f8acfc8b0b5c0071b804423bcf53fda26c7c12.
Diffstat (limited to 'yt/cpp/mapreduce/client/py_helpers.cpp')
-rw-r--r--yt/cpp/mapreduce/client/py_helpers.cpp112
1 files changed, 0 insertions, 112 deletions
diff --git a/yt/cpp/mapreduce/client/py_helpers.cpp b/yt/cpp/mapreduce/client/py_helpers.cpp
deleted file mode 100644
index 3072449866e..00000000000
--- a/yt/cpp/mapreduce/client/py_helpers.cpp
+++ /dev/null
@@ -1,112 +0,0 @@
-#include "py_helpers.h"
-
-#include "client.h"
-#include "operation.h"
-#include "transaction.h"
-
-#include <yt/cpp/mapreduce/interface/client.h>
-#include <yt/cpp/mapreduce/interface/fluent.h>
-
-#include <yt/cpp/mapreduce/common/retry_lib.h>
-#include <yt/cpp/mapreduce/common/helpers.h>
-
-#include <library/cpp/yson/node/node_io.h>
-
-#include <util/generic/hash_set.h>
-
-namespace NYT {
-
-using namespace NDetail;
-
-////////////////////////////////////////////////////////////////////////////////
-
-IStructuredJobPtr ConstructJob(const TString& jobName, const TString& state)
-{
- auto node = TNode();
- if (!state.empty()) {
- node = NodeFromYsonString(state);
- }
- return TJobFactory::Get()->GetConstructingFunction(jobName.data())(node);
-}
-
-TString GetJobStateString(const IStructuredJob& job)
-{
- TString result;
- {
- TStringOutput output(result);
- job.Save(output);
- output.Finish();
- }
- return result;
-}
-
-TStructuredJobTableList NodeToStructuredTablePaths(const TNode& node, const TOperationPreparer& preparer)
-{
- int intermediateTableCount = 0;
- TVector<TRichYPath> paths;
- for (const auto& inputNode : node.AsList()) {
- if (inputNode.IsNull()) {
- ++intermediateTableCount;
- } else {
- paths.emplace_back(inputNode.AsString());
- }
- }
- paths = NRawClient::CanonizeYPaths(/* retryPolicy */ nullptr, preparer.GetContext(), paths);
- TStructuredJobTableList result(intermediateTableCount, TStructuredJobTable::Intermediate(TUnspecifiedTableStructure()));
- for (const auto& path : paths) {
- result.emplace_back(TStructuredJobTable{TUnspecifiedTableStructure(), path});
- }
- return result;
-}
-
-TString GetIOInfo(
- const IStructuredJob& job,
- const TCreateClientOptions& options,
- const TString& cluster,
- const TString& transactionId,
- const TString& inputPaths,
- const TString& outputPaths,
- const TString& neededColumns)
-{
- auto client = NDetail::CreateClientImpl(cluster, options);
- TOperationPreparer preparer(client, GetGuid(transactionId));
-
- auto structuredInputs = NodeToStructuredTablePaths(NodeFromYsonString(inputPaths), preparer);
- auto structuredOutputs = NodeToStructuredTablePaths(NodeFromYsonString(outputPaths), preparer);
-
- auto neededColumnsNode = NodeFromYsonString(neededColumns);
- THashSet<TString> columnsUsedInOperations;
- for (const auto& columnNode : neededColumnsNode.AsList()) {
- columnsUsedInOperations.insert(columnNode.AsString());
- }
-
- auto operationIo = CreateSimpleOperationIoHelper(
- job,
- preparer,
- TOperationOptions(),
- std::move(structuredInputs),
- std::move(structuredOutputs),
- TUserJobFormatHints(),
- ENodeReaderFormat::Yson,
- columnsUsedInOperations);
-
- return BuildYsonStringFluently().BeginMap()
- .Item("input_format").Value(operationIo.InputFormat.Config)
- .Item("output_format").Value(operationIo.OutputFormat.Config)
- .Item("input_table_paths").List(operationIo.Inputs)
- .Item("output_table_paths").List(operationIo.Outputs)
- .Item("small_files").DoListFor(
- operationIo.JobFiles.begin(),
- operationIo.JobFiles.end(),
- [] (TFluentList fluent, auto fileIt) {
- fluent.Item().BeginMap()
- .Item("file_name").Value(fileIt->FileName)
- .Item("data").Value(fileIt->Data)
- .EndMap();
- })
- .EndMap();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-} // namespace NYT