diff options
author | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
---|---|---|
committer | max42 <max42@yandex-team.com> | 2023-07-29 00:02:16 +0300 |
commit | 73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch) | |
tree | 188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /yt/cpp/mapreduce/client/structured_table_formats.h | |
parent | 528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff) | |
download | ydb-73b89de71748a21e102d27b9f3ed1bf658766cb5.tar.gz |
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build.
Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'yt/cpp/mapreduce/client/structured_table_formats.h')
-rw-r--r-- | yt/cpp/mapreduce/client/structured_table_formats.h | 146 |
1 files changed, 146 insertions, 0 deletions
diff --git a/yt/cpp/mapreduce/client/structured_table_formats.h b/yt/cpp/mapreduce/client/structured_table_formats.h new file mode 100644 index 0000000000..27d980c587 --- /dev/null +++ b/yt/cpp/mapreduce/client/structured_table_formats.h @@ -0,0 +1,146 @@ +#pragma once + +#include <yt/cpp/mapreduce/interface/fwd.h> + +#include <yt/cpp/mapreduce/interface/config.h> +#include <yt/cpp/mapreduce/interface/operation.h> + +#include <yt/cpp/mapreduce/common/fwd.h> + +#include <yt/cpp/mapreduce/http/context.h> +#include <yt/cpp/mapreduce/http/requests.h> + +#include <utility> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TMaybe<TNode> GetCommonTableFormat( + const TVector<TMaybe<TNode>>& formats); + +TMaybe<TNode> GetTableFormat( + const IClientRetryPolicyPtr& clientRetryPolicy, + const TClientContext& context, + const TTransactionId& transactionId, + const TRichYPath& path); + +TMaybe<TNode> GetTableFormats( + const IClientRetryPolicyPtr& clientRetryPolicy, + const TClientContext& context, + const TTransactionId& transactionId, + const TVector<TRichYPath>& paths); + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +//////////////////////////////////////////////////////////////////////////////// + +enum class EIODirection +{ + Input, + Output, +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TSmallJobFile +{ + TString FileName; + TString Data; +}; + +//////////////////////////////////////////////////////////////////////////////// + +// Table that is used while preparing operation formats. Can be real table or intermediate +struct TStructuredJobTable +{ + TTableStructure Description; + // Might be null for intermediate tables in MapReduce operation + TMaybe<TRichYPath> RichYPath; + + static TStructuredJobTable Intermediate(TTableStructure description) + { + return TStructuredJobTable{std::move(description), Nothing()}; + } +}; +using TStructuredJobTableList = TVector<TStructuredJobTable>; +TString JobTablePathString(const TStructuredJobTable& jobTable); +TStructuredJobTableList ToStructuredJobTableList(const TVector<TStructuredTablePath>& tableList); + +TStructuredJobTableList CanonizeStructuredTableList(const TClientContext& context, const TVector<TStructuredTablePath>& tableList); +TVector<TRichYPath> GetPathList( + const TStructuredJobTableList& tableList, + const TMaybe<TVector<TTableSchema>>& schemaInferenceResult, + bool inferSchema); + +//////////////////////////////////////////////////////////////////////////////// + +class TFormatBuilder +{ +private: + struct TFormatSwitcher; + +public: + TFormatBuilder( + IClientRetryPolicyPtr clientRetryPolicy, + TClientContext context, + TTransactionId transactionId, + TOperationOptions operationOptions); + + std::pair<TFormat, TMaybe<TSmallJobFile>> CreateFormat( + const IStructuredJob& job, + const EIODirection& direction, + const TStructuredJobTableList& structuredTableList, + const TMaybe<TFormatHints>& formatHints, + ENodeReaderFormat nodeReaderFormat, + bool allowFormatFromTableAttribute); + + std::pair<TFormat, TMaybe<TSmallJobFile>> CreateVoidFormat( + const IStructuredJob& job, + const EIODirection& direction, + const TStructuredJobTableList& structuredTableList, + const TMaybe<TFormatHints>& formatHints, + ENodeReaderFormat nodeReaderFormat, + bool allowFormatFromTableAttribute); + + std::pair<TFormat, TMaybe<TSmallJobFile>> CreateYamrFormat( + const IStructuredJob& job, + const EIODirection& direction, + const TStructuredJobTableList& structuredTableList, + const TMaybe<TFormatHints>& formatHints, + ENodeReaderFormat nodeReaderFormat, + bool allowFormatFromTableAttribute); + + std::pair<TFormat, TMaybe<TSmallJobFile>> CreateNodeFormat( + const IStructuredJob& job, + const EIODirection& direction, + const TStructuredJobTableList& structuredTableList, + const TMaybe<TFormatHints>& formatHints, + ENodeReaderFormat nodeReaderFormat, + bool allowFormatFromTableAttribute); + + std::pair<TFormat, TMaybe<TSmallJobFile>> CreateProtobufFormat( + const IStructuredJob& job, + const EIODirection& direction, + const TStructuredJobTableList& structuredTableList, + const TMaybe<TFormatHints>& formatHints, + ENodeReaderFormat nodeReaderFormat, + bool allowFormatFromTableAttribute); + +private: + const IClientRetryPolicyPtr ClientRetryPolicy_; + const TClientContext Context_; + const TTransactionId TransactionId_; + const TOperationOptions OperationOptions_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TMaybe<TTableSchema> GetTableSchema(const TTableStructure& tableStructure); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NDetail +} // namespace NYT |