summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorauzhegov <[email protected]>2023-09-29 15:08:20 +0300
committerauzhegov <[email protected]>2023-09-29 15:34:32 +0300
commitaea193b8f0b37198b710ab6b7511c13be4edd89b (patch)
treec3c87b6faed7c46f97b8fbb61cd808432e8016eb
parent081921c5f531f3a011f94cd1695b6c8a67335299 (diff)
Limit max result set per partition
-rw-r--r--ydb/library/yql/providers/common/proto/gateways_config.proto1
-rw-r--r--ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp4
-rw-r--r--ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp5
-rw-r--r--ydb/library/yql/providers/s3/provider/yql_s3_settings.h2
4 files changed, 12 insertions, 0 deletions
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 4f9184ba472..318a8fa9c13 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -397,6 +397,7 @@ message TS3GatewayConfig {
optional uint64 ListingCallbackPerThreadQueueSize = 13;
optional uint64 RegexpCacheSize = 14;
optional uint64 GeneratorPathsLimit = 15;
+ optional uint64 MaxListingResultSizePerPartition = 16;
repeated TAttr DefaultSettings = 100;
}
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
index 0535de0f960..995b416723e 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
@@ -587,6 +587,10 @@ private:
if (!s3ParseSettingsBase.Paths().Empty()) {
resultSetLimitPerPath /= s3ParseSettingsBase.Paths().Size();
}
+ resultSetLimitPerPath =
+ std::min(resultSetLimitPerPath,
+ State_->Configuration->MaxDiscoveryFilesPerDirectory.Get().GetOrElse(
+ State_->Configuration->MaxListingResultSizePerPhysicalPartition));
for (auto path : s3ParseSettingsBase.Paths()) {
NS3Details::TPathList directories;
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp
index af2346806b0..411c6f3ada2 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp
@@ -21,6 +21,7 @@ TS3Configuration::TS3Configuration()
REGISTER_SETTING(*this, UseBlocksSource);
REGISTER_SETTING(*this, AtomicUploadCommit);
REGISTER_SETTING(*this, UseConcurrentDirectoryLister);
+ REGISTER_SETTING(*this, MaxDiscoveryFilesPerDirectory).Lower(1);
}
TS3Settings::TConstPtr TS3Configuration::Snapshot() const {
@@ -66,6 +67,10 @@ void TS3Configuration::Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeA
config.HasAllowConcurrentListings() ? config.GetAllowConcurrentListings() : false;
GeneratorPathsLimit =
config.HasGeneratorPathsLimit() ? config.GetGeneratorPathsLimit() : 50'000;
+ MaxListingResultSizePerPhysicalPartition =
+ config.HasMaxListingResultSizePerPartition()
+ ? config.GetMaxListingResultSizePerPartition()
+ : 1'000;
TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
for (auto& cluster: config.GetClusterMapping()) {
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h
index 8573192b33f..4f4a8734d68 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h
@@ -23,6 +23,7 @@ struct TS3Settings {
NCommon::TConfSetting<bool, false> UseBlocksSource; // Use blocks source (if exists) for scalar MKQL mode
NCommon::TConfSetting<bool, false> AtomicUploadCommit; // Commit each file independently, w/o transaction semantic over all files
NCommon::TConfSetting<bool, false> UseConcurrentDirectoryLister;
+ NCommon::TConfSetting<ui64, false> MaxDiscoveryFilesPerDirectory;
};
struct TS3ClusterSettings {
@@ -59,6 +60,7 @@ struct TS3Configuration : public TS3Settings, public NCommon::TSettingDispatcher
bool AllowConcurrentListings = false;
ui64 GeneratorPathsLimit = 0;
bool WriteThroughDqIntegration = false;
+ ui64 MaxListingResultSizePerPhysicalPartition;
};
} // NYql