aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormrlolthe1st <mrlolthe1st@yandex-team.com>2023-07-31 16:42:43 +0300
committermrlolthe1st <mrlolthe1st@yandex-team.com>2023-07-31 16:42:43 +0300
commit4fcb9ce9ffcb5ba0f31ceca0eb4f3696fe15e16b (patch)
tree5d1a4eba5f3a750542acf11625b2ba25770b7379
parent595034cc33a4dc2d0d2057bd0c9e36342b52184f (diff)
downloadydb-4fcb9ce9ffcb5ba0f31ceca0eb4f3696fe15e16b.tar.gz
YQL-16232: Check chunks count earlier
YQL-16232: Check chunks count earlier
-rw-r--r--ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp14
1 files changed, 6 insertions, 8 deletions
diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp
index 32b40de6b1..1c21299802 100644
--- a/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp
+++ b/ydb/library/yql/providers/yt/provider/yql_yt_dq_integration.cpp
@@ -278,7 +278,7 @@ public:
AddInfo(ctx, TStringBuilder() << "disabled for cluster " << cluster, skipIssues);
return false;
}
-
+ ui64 chunksCount = 0ull;
for (auto section: maybeRead.Cast().Input()) {
if (HasSettingsExcept(maybeRead.Cast().Input().Item(0).Settings().Ref(), DqReadSupportedSettings)) {
TStringBuilder info;
@@ -320,9 +320,14 @@ public:
AddInfo(ctx, "table with QB2 premapper", skipIssues);
return false;
}
+ chunksCount += tableInfo->Stat->ChunkCount;
}
}
}
+ if (auto maxChunks = State_->Configuration->MaxChunksForDqRead.Get().GetOrElse(DEFAULT_MAX_CHUNKS_FOR_DQ_READ); chunksCount > maxChunks) {
+ AddInfo(ctx, "table with too many chunks", skipIssues);
+ return false;
+ }
return true;
}
AddInfo(ctx, TStringBuilder() << "unsupported callable: " << node.Content(), skipIssues);
@@ -334,7 +339,6 @@ public:
ui64 dataSize = 0;
bool hasErasure = false;
- ui64 chunksCount = 0;
auto cluster = maybeRead.Cast().DataSource().Cluster().StringValue();
const auto canUseYtPartitioningApi = State_->Configuration->_EnableYtPartitioning.Get(cluster).GetOrElse(false);
@@ -362,7 +366,6 @@ public:
if (tableInfo->Meta->Attrs.Value("erasure_codec", "none") != "none") {
hasErasure = true;
}
- chunksCount += tableInfo->Stat->ChunkCount;
}
groupIdPathInfos.back().emplace_back(pathInfo);
}
@@ -381,11 +384,6 @@ public:
}
}
- if (auto maxChunks = State_->Configuration->MaxChunksForDqRead.Get().GetOrElse(DEFAULT_MAX_CHUNKS_FOR_DQ_READ); chunksCount > maxChunks) {
- AddErrorWrap(ctx, node.Pos(), "table with too many chunks");
- return Nothing();
- }
-
return dataSize;
}
AddErrorWrap(ctx, node.Pos(), TStringBuilder() << "unsupported callable: " << node.Content());