diff options
| author | mrlolthe1st <mrlolthe1st@yandex-team.com> | 2024-12-28 13:09:16 +0300 |
|---|---|---|
| committer | mrlolthe1st <mrlolthe1st@yandex-team.com> | 2024-12-28 13:27:37 +0300 |
| commit | 4af51e819db87713b847e3e1f3e080948ba3eb67 (patch) | |
| tree | c46e662a5f2d82c0b8fbf6be1b90d97b3dd9e977 | |
| parent | d679a6082d9a96a0e355545a41e0d06bbc3dfbd1 (diff) | |
| download | ydb-4af51e819db87713b847e3e1f3e080948ba3eb67.tar.gz | |
BlockReader: allow non-native json, do not use casts on some types
BlockReader: allow non-native json, do not use casts on some types
commit_hash:5b24acf4ad08b6b328dc530b0c0d5b859715f4fa
| -rw-r--r-- | yt/yql/providers/yt/codec/yt_arrow_converter.cpp | 6 | ||||
| -rw-r--r-- | yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp | 11 |
2 files changed, 14 insertions, 3 deletions
diff --git a/yt/yql/providers/yt/codec/yt_arrow_converter.cpp b/yt/yql/providers/yt/codec/yt_arrow_converter.cpp index 11b512ce2b..673b2876ea 100644 --- a/yt/yql/providers/yt/codec/yt_arrow_converter.cpp +++ b/yt/yql/providers/yt/codec/yt_arrow_converter.cpp @@ -681,11 +681,11 @@ std::unique_ptr<IYtColumnConverter> MakeYtColumnConverter(TType* type, const NUd case NUdf::EDataSlot::Bool: // YT type for bool is arrow::Type::BOOL, but yql type is arrow::Type::UINT8 return std::make_unique<TTopLevelSimpleCastConverter<arrow::Type::BOOL>>(std::move(settings)); - case NUdf::EDataSlot::String: case NUdf::EDataSlot::Json: - case NUdf::EDataSlot::Yson: // Yson there is top-level optional - // YT type for Yson, Json, String is arrow::Type::BINARY, but yql type is arrow::Type::String return std::make_unique<TTopLevelSimpleCastConverter<arrow::Type::BINARY>>(std::move(settings)); + case NUdf::EDataSlot::JsonDocument: + case NUdf::EDataSlot::String: + case NUdf::EDataSlot::Yson: case NUdf::EDataSlot::Utf8: case NUdf::EDataSlot::Double: case NUdf::EDataSlot::Int8: diff --git a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp index b8891b1208..4d8c967e5f 100644 --- a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp @@ -521,6 +521,17 @@ public: auto section = sectionList.Item(i); auto paths = section.Paths(); for (const auto& path : section.Paths()) { + auto rowSpec = TYtTableBaseInfo::GetRowSpec(path.Table()); + if (!rowSpec) { + BlockReaderAddInfo(ctx, ctx.GetPosition(node.Pos()), "table without rowspec"); + return false; + } + + if (rowSpec->GetNativeYtTypeFlags() & NTCF_JSON) { + BlockReaderAddInfo(ctx, ctx.GetPosition(node.Pos()), "native json is not supported yet by arrow encoder at YT side"); + return false; + } + if (!IsYtTableSuitableForArrowInput(path.Table(), [&ctx, &node](const TString& msg) { BlockReaderAddInfo(ctx, ctx.GetPosition(node.Pos()), msg); })) { |