diff options
author | mrlolthe1st <mrlolthe1st@yandex-team.com> | 2023-07-31 16:42:43 +0300 |
---|---|---|
committer | mrlolthe1st <mrlolthe1st@yandex-team.com> | 2023-07-31 16:42:43 +0300 |
commit | 4fcb9ce9ffcb5ba0f31ceca0eb4f3696fe15e16b (patch) | |
tree | 5d1a4eba5f3a750542acf11625b2ba25770b7379 | |
parent | 595034cc33a4dc2d0d2057bd0c9e36342b52184f (diff) | |
download | ydb-4fcb9ce9ffcb5ba0f31ceca0eb4f3696fe15e16b.tar.gz |
YQL-16232: Check chunks count earlier
YQL-16232: Check chunks count earlier
-rw-r--r-- | ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp | 14 |
1 files changed, 6 insertions, 8 deletions
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp index 32b40de6b1..1c21299802 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp @@ -278,7 +278,7 @@ public: AddInfo(ctx, TStringBuilder() << "disabled for cluster " << cluster, skipIssues); return false; } - + ui64 chunksCount = 0ull; for (auto section: maybeRead.Cast().Input()) { if (HasSettingsExcept(maybeRead.Cast().Input().Item(0).Settings().Ref(), DqReadSupportedSettings)) { TStringBuilder info; @@ -320,9 +320,14 @@ public: AddInfo(ctx, "table with QB2 premapper", skipIssues); return false; } + chunksCount += tableInfo->Stat->ChunkCount; } } } + if (auto maxChunks = State_->Configuration->MaxChunksForDqRead.Get().GetOrElse(DEFAULT_MAX_CHUNKS_FOR_DQ_READ); chunksCount > maxChunks) { + AddInfo(ctx, "table with too many chunks", skipIssues); + return false; + } return true; } AddInfo(ctx, TStringBuilder() << "unsupported callable: " << node.Content(), skipIssues); @@ -334,7 +339,6 @@ public: ui64 dataSize = 0; bool hasErasure = false; - ui64 chunksCount = 0; auto cluster = maybeRead.Cast().DataSource().Cluster().StringValue(); const auto canUseYtPartitioningApi = State_->Configuration->_EnableYtPartitioning.Get(cluster).GetOrElse(false); @@ -362,7 +366,6 @@ public: if (tableInfo->Meta->Attrs.Value("erasure_codec", "none") != "none") { hasErasure = true; } - chunksCount += tableInfo->Stat->ChunkCount; } groupIdPathInfos.back().emplace_back(pathInfo); } @@ -381,11 +384,6 @@ public: } } - if (auto maxChunks = State_->Configuration->MaxChunksForDqRead.Get().GetOrElse(DEFAULT_MAX_CHUNKS_FOR_DQ_READ); chunksCount > maxChunks) { - AddErrorWrap(ctx, node.Pos(), "table with too many chunks"); - return Nothing(); - } - return dataSize; } AddErrorWrap(ctx, node.Pos(), TStringBuilder() << "unsupported callable: " << node.Content()); |