Limit max result set per partition

4 files changed, 12 insertions, 0 deletions

diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 4f9184ba472..318a8fa9c13 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -397,6 +397,7 @@ message TS3GatewayConfig {
     optional uint64 ListingCallbackPerThreadQueueSize = 13;
     optional uint64 RegexpCacheSize = 14;
     optional uint64 GeneratorPathsLimit = 15;
+    optional uint64 MaxListingResultSizePerPartition = 16;
 
     repeated TAttr DefaultSettings = 100;
 }
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
index 0535de0f960..995b416723e 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
@@ -587,6 +587,10 @@ private:
         if (!s3ParseSettingsBase.Paths().Empty()) {
             resultSetLimitPerPath /= s3ParseSettingsBase.Paths().Size();
         }
+        resultSetLimitPerPath =
+            std::min(resultSetLimitPerPath,
+                     State_->Configuration->MaxDiscoveryFilesPerDirectory.Get().GetOrElse(
+                         State_->Configuration->MaxListingResultSizePerPhysicalPartition));
 
         for (auto path : s3ParseSettingsBase.Paths()) {
             NS3Details::TPathList directories;
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp
index af2346806b0..411c6f3ada2 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.cpp
@@ -21,6 +21,7 @@ TS3Configuration::TS3Configuration()
     REGISTER_SETTING(*this, UseBlocksSource);
     REGISTER_SETTING(*this, AtomicUploadCommit);
     REGISTER_SETTING(*this, UseConcurrentDirectoryLister);
+    REGISTER_SETTING(*this, MaxDiscoveryFilesPerDirectory).Lower(1);
 }
 
 TS3Settings::TConstPtr TS3Configuration::Snapshot() const {
@@ -66,6 +67,10 @@ void TS3Configuration::Init(const TS3GatewayConfig& config, TIntrusivePtr<TTypeA
         config.HasAllowConcurrentListings() ? config.GetAllowConcurrentListings() : false;
     GeneratorPathsLimit =
         config.HasGeneratorPathsLimit() ? config.GetGeneratorPathsLimit() : 50'000;
+    MaxListingResultSizePerPhysicalPartition =
+        config.HasMaxListingResultSizePerPartition()
+            ? config.GetMaxListingResultSizePerPartition()
+            : 1'000;
 
     TVector<TString> clusters(Reserve(config.ClusterMappingSize()));
     for (auto& cluster: config.GetClusterMapping()) {
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h
index 8573192b33f..4f4a8734d68 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_settings.h
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_settings.h
@@ -23,6 +23,7 @@ struct TS3Settings {
     NCommon::TConfSetting<bool, false> UseBlocksSource;            // Use blocks source (if exists) for scalar MKQL mode
     NCommon::TConfSetting<bool, false> AtomicUploadCommit;         // Commit each file independently, w/o transaction semantic over all files
     NCommon::TConfSetting<bool, false> UseConcurrentDirectoryLister;
+    NCommon::TConfSetting<ui64, false> MaxDiscoveryFilesPerDirectory;
 };
 
 struct TS3ClusterSettings {
@@ -59,6 +60,7 @@ struct TS3Configuration : public TS3Settings, public NCommon::TSettingDispatcher
     bool AllowConcurrentListings = false;
     ui64 GeneratorPathsLimit = 0;
     bool WriteThroughDqIntegration = false;
+    ui64 MaxListingResultSizePerPhysicalPartition;
 };
 
 } // NYql