aboutsummaryrefslogtreecommitdiffstats
path: root/yt/cpp/mapreduce/client/structured_table_formats.h
diff options
context:
space:
mode:
authormax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
committermax42 <max42@yandex-team.com>2023-07-29 00:02:16 +0300
commit73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch)
tree188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/cpp/mapreduce/client/structured_table_formats.h
parent528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff)
downloadydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build. Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/cpp/mapreduce/client/structured_table_formats.h')
-rw-r--r--yt/cpp/mapreduce/client/structured_table_formats.h146
1 files changed, 146 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/client/structured_table_formats.h b/yt/cpp/mapreduce/client/structured_table_formats.h
new file mode 100644
index 0000000000..27d980c587
--- /dev/null
+++ b/yt/cpp/mapreduce/client/structured_table_formats.h
@@ -0,0 +1,146 @@
+#pragma once
+
+#include <yt/cpp/mapreduce/interface/fwd.h>
+
+#include <yt/cpp/mapreduce/interface/config.h>
+#include <yt/cpp/mapreduce/interface/operation.h>
+
+#include <yt/cpp/mapreduce/common/fwd.h>
+
+#include <yt/cpp/mapreduce/http/context.h>
+#include <yt/cpp/mapreduce/http/requests.h>
+
+#include <utility>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TMaybe<TNode> GetCommonTableFormat(
+ const TVector<TMaybe<TNode>>& formats);
+
+TMaybe<TNode> GetTableFormat(
+ const IClientRetryPolicyPtr& clientRetryPolicy,
+ const TClientContext& context,
+ const TTransactionId& transactionId,
+ const TRichYPath& path);
+
+TMaybe<TNode> GetTableFormats(
+ const IClientRetryPolicyPtr& clientRetryPolicy,
+ const TClientContext& context,
+ const TTransactionId& transactionId,
+ const TVector<TRichYPath>& paths);
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NDetail {
+
+////////////////////////////////////////////////////////////////////////////////
+
+enum class EIODirection
+{
+ Input,
+ Output,
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TSmallJobFile
+{
+ TString FileName;
+ TString Data;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Table that is used while preparing operation formats. Can be real table or intermediate
+struct TStructuredJobTable
+{
+ TTableStructure Description;
+ // Might be null for intermediate tables in MapReduce operation
+ TMaybe<TRichYPath> RichYPath;
+
+ static TStructuredJobTable Intermediate(TTableStructure description)
+ {
+ return TStructuredJobTable{std::move(description), Nothing()};
+ }
+};
+using TStructuredJobTableList = TVector<TStructuredJobTable>;
+TString JobTablePathString(const TStructuredJobTable& jobTable);
+TStructuredJobTableList ToStructuredJobTableList(const TVector<TStructuredTablePath>& tableList);
+
+TStructuredJobTableList CanonizeStructuredTableList(const TClientContext& context, const TVector<TStructuredTablePath>& tableList);
+TVector<TRichYPath> GetPathList(
+ const TStructuredJobTableList& tableList,
+ const TMaybe<TVector<TTableSchema>>& schemaInferenceResult,
+ bool inferSchema);
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TFormatBuilder
+{
+private:
+ struct TFormatSwitcher;
+
+public:
+ TFormatBuilder(
+ IClientRetryPolicyPtr clientRetryPolicy,
+ TClientContext context,
+ TTransactionId transactionId,
+ TOperationOptions operationOptions);
+
+ std::pair<TFormat, TMaybe<TSmallJobFile>> CreateFormat(
+ const IStructuredJob& job,
+ const EIODirection& direction,
+ const TStructuredJobTableList& structuredTableList,
+ const TMaybe<TFormatHints>& formatHints,
+ ENodeReaderFormat nodeReaderFormat,
+ bool allowFormatFromTableAttribute);
+
+ std::pair<TFormat, TMaybe<TSmallJobFile>> CreateVoidFormat(
+ const IStructuredJob& job,
+ const EIODirection& direction,
+ const TStructuredJobTableList& structuredTableList,
+ const TMaybe<TFormatHints>& formatHints,
+ ENodeReaderFormat nodeReaderFormat,
+ bool allowFormatFromTableAttribute);
+
+ std::pair<TFormat, TMaybe<TSmallJobFile>> CreateYamrFormat(
+ const IStructuredJob& job,
+ const EIODirection& direction,
+ const TStructuredJobTableList& structuredTableList,
+ const TMaybe<TFormatHints>& formatHints,
+ ENodeReaderFormat nodeReaderFormat,
+ bool allowFormatFromTableAttribute);
+
+ std::pair<TFormat, TMaybe<TSmallJobFile>> CreateNodeFormat(
+ const IStructuredJob& job,
+ const EIODirection& direction,
+ const TStructuredJobTableList& structuredTableList,
+ const TMaybe<TFormatHints>& formatHints,
+ ENodeReaderFormat nodeReaderFormat,
+ bool allowFormatFromTableAttribute);
+
+ std::pair<TFormat, TMaybe<TSmallJobFile>> CreateProtobufFormat(
+ const IStructuredJob& job,
+ const EIODirection& direction,
+ const TStructuredJobTableList& structuredTableList,
+ const TMaybe<TFormatHints>& formatHints,
+ ENodeReaderFormat nodeReaderFormat,
+ bool allowFormatFromTableAttribute);
+
+private:
+ const IClientRetryPolicyPtr ClientRetryPolicy_;
+ const TClientContext Context_;
+ const TTransactionId TransactionId_;
+ const TOperationOptions OperationOptions_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TMaybe<TTableSchema> GetTableSchema(const TTableStructure& tableStructure);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NDetail
+} // namespace NYT