diff options
author | auzhegov <[email protected]> | 2023-09-29 15:08:20 +0300 |
---|---|---|
committer | auzhegov <[email protected]> | 2023-09-29 15:34:32 +0300 |
commit | aea193b8f0b37198b710ab6b7511c13be4edd89b (patch) | |
tree | c3c87b6faed7c46f97b8fbb61cd808432e8016eb | |
parent | 081921c5f531f3a011f94cd1695b6c8a67335299 (diff) |
Limit max result set per partition
4 files changed, 12 insertions, 0 deletions
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 4f9184ba472..318a8fa9c13 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -397,6 +397,7 @@ message TS3GatewayConfig { optional uint64 ListingCallbackPerThreadQueueSize = 13; optional uint64 RegexpCacheSize = 14; optional uint64 GeneratorPathsLimit = 15; + optional uint64 MaxListingResultSizePerPartition = 16; repeated TAttr DefaultSettings = 100; } diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp index 0535de0f960..995b416723e 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp @@ -587,6 +587,10 @@ private: if (!s3ParseSettingsBase.Paths().Empty()) { resultSetLimitPerPath /= s3ParseSettingsBase.Paths().Size(); } + resultSetLimitPerPath = + std::min(resultSetLimitPerPath, + State_->Configuration->MaxDiscoveryFilesPerDirectory.Get().GetOrElse( + State_->Configuration->MaxListingResultSizePerPhysicalPartition)); for (auto path : s3ParseSettingsBase.Paths()) { NS3Details::TPathList directories; diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp index af2346806b0..411c6f3ada2 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp @@ -21,6 +21,7 @@ TS3Configuration::TS3Configuration() REGISTER_SETTING(*this, UseBlocksSource); REGISTER_SETTING(*this, AtomicUploadCommit); REGISTER_SETTING(*this, UseConcurrentDirectoryLister); + REGISTER_SETTING(*this, MaxDiscoveryFilesPerDirectory).Lower(1); } TS3Settings::TConstPtr TS3Configuration::Snapshot() const { @@ -66,6 +67,10 @@ void TS3Configuration::Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeA config.HasAllowConcurrentListings() ? config.GetAllowConcurrentListings() : false; GeneratorPathsLimit = config.HasGeneratorPathsLimit() ? config.GetGeneratorPathsLimit() : 50'000; + MaxListingResultSizePerPhysicalPartition = + config.HasMaxListingResultSizePerPartition() + ? config.GetMaxListingResultSizePerPartition() + : 1'000; TVector<TString> clusters(Reserve(config.ClusterMappingSize())); for (auto& cluster: config.GetClusterMapping()) { diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h index 8573192b33f..4f4a8734d68 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h +++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h @@ -23,6 +23,7 @@ struct TS3Settings { NCommon::TConfSetting<bool, false> UseBlocksSource; // Use blocks source (if exists) for scalar MKQL mode NCommon::TConfSetting<bool, false> AtomicUploadCommit; // Commit each file independently, w/o transaction semantic over all files NCommon::TConfSetting<bool, false> UseConcurrentDirectoryLister; + NCommon::TConfSetting<ui64, false> MaxDiscoveryFilesPerDirectory; }; struct TS3ClusterSettings { @@ -59,6 +60,7 @@ struct TS3Configuration : public TS3Settings, public NCommon::TSettingDispatcher bool AllowConcurrentListings = false; ui64 GeneratorPathsLimit = 0; bool WriteThroughDqIntegration = false; + ui64 MaxListingResultSizePerPhysicalPartition; }; } // NYql |