diff options
| author | aneporada <[email protected]> | 2025-05-20 13:16:51 +0300 |
|---|---|---|
| committer | aneporada <[email protected]> | 2025-05-20 13:33:08 +0300 |
| commit | 587d6bb291fbf231d6440d01326d4fd3387bcf77 (patch) | |
| tree | e43185c07b55f695d2387dd04661cde18c86754e | |
| parent | ae913bccb4fd8a8ba5f536c8afe622e2ad229fbc (diff) | |
Refactor TConfSetting
commit_hash:2ca0a7e131758036bd15ba9e58dce176b9eff6c9
| -rw-r--r-- | yql/essentials/providers/common/config/yql_dispatch.h | 39 | ||||
| -rw-r--r-- | yql/essentials/providers/common/config/yql_setting.h | 19 | ||||
| -rw-r--r-- | yt/yql/providers/yt/common/yql_yt_settings.h | 470 |
3 files changed, 301 insertions, 227 deletions
diff --git a/yql/essentials/providers/common/config/yql_dispatch.h b/yql/essentials/providers/common/config/yql_dispatch.h index 97630e857de..bec237df32e 100644 --- a/yql/essentials/providers/common/config/yql_dispatch.h +++ b/yql/essentials/providers/common/config/yql_dispatch.h @@ -76,6 +76,23 @@ TParser<TInstant> GetDefaultParser<TInstant>(); YQL_PRIMITIVE_SETTING_PARSER_TYPES(YQL_DECLARE_SETTING_PARSER) YQL_CONTAINER_SETTING_PARSER_TYPES(YQL_DECLARE_SETTING_PARSER) +#ifdef YQL_BETTER_CONF_SETTING_API +template<typename TType> +TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, NCommon::EConfSettingType::StaticPerCluster>& setting, const TString& cluster) { + return setting.Get(cluster); +} + +template<typename TType> +TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, NCommon::EConfSettingType::Dynamic>& setting, const TString& cluster) { + return setting.Get(cluster); +} + +template<typename TType> +TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, NCommon::EConfSettingType::Static>& setting, const TString& cluster) { + Y_UNUSED(cluster); + return setting.Get(); +} +#else template<typename TType> TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, true, false>& setting, const TString& cluster) { return setting.Get(cluster); @@ -96,6 +113,7 @@ TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, false, false>& setting Y_UNUSED(cluster); return setting.Get(); } +#endif } @@ -135,15 +153,23 @@ public: TString Name_; }; +#ifdef YQL_BETTER_CONF_SETTING_API + template <typename TType, EConfSettingType SettingType> +#else template <typename TType, bool RUNTIME, bool PERCLUSTER> - class TSettingHandlerImpl: public TSettingHandler { +#endif + class TSettingHandlerImpl : public TSettingHandler { public: using TValueCallback = std::function<void(const TString&, TType)>; private: friend class TSettingDispatcher; +#ifdef YQL_BETTER_CONF_SETTING_API + TSettingHandlerImpl(const TString& name, TConfSetting<TType, SettingType>& setting) +#else TSettingHandlerImpl(const TString& name, TConfSetting<TType, RUNTIME, PERCLUSTER>& setting) +#endif : TSettingHandler(name) , Setting_(setting) , Parser_(::NYql::NPrivate::GetDefaultParser<TType>()) @@ -322,8 +348,13 @@ public: } private: +#ifdef YQL_BETTER_CONF_SETTING_API + TConfSetting<TType, SettingType>& Setting_; + TMaybe<TConfSetting<TType, SettingType>> Defaul_; +#else TConfSetting<TType, RUNTIME, PERCLUSTER>& Setting_; TMaybe<TConfSetting<TType, RUNTIME, PERCLUSTER>> Defaul_; +#endif ::NYql::NPrivate::TParser<TType> Parser_; TValueCallback ValueSetter_; TVector<TValueCallback> Validators_; @@ -349,9 +380,15 @@ public: ValidClusters.insert(cluster); } +#ifdef YQL_BETTER_CONF_SETTING_API + template <typename TType, EConfSettingType SettingType> + TSettingHandlerImpl<TType, SettingType>& AddSetting(const TString& name, TConfSetting<TType, SettingType>& setting) { + TIntrusivePtr<TSettingHandlerImpl<TType, SettingType>> handler = new TSettingHandlerImpl<TType, SettingType>(name, setting); +#else template <typename TType, bool RUNTIME, bool PERCLUSTER> TSettingHandlerImpl<TType, RUNTIME, PERCLUSTER>& AddSetting(const TString& name, TConfSetting<TType, RUNTIME, PERCLUSTER>& setting) { TIntrusivePtr<TSettingHandlerImpl<TType, RUNTIME, PERCLUSTER>> handler = new TSettingHandlerImpl<TType, RUNTIME, PERCLUSTER>(name, setting); +#endif if (!Handlers.insert({NormalizeName(name), handler}).second) { ythrow yexception() << "Duplicate configuration setting name " << name.Quote(); } diff --git a/yql/essentials/providers/common/config/yql_setting.h b/yql/essentials/providers/common/config/yql_setting.h index 78e90ca0c5a..3b2110fbf28 100644 --- a/yql/essentials/providers/common/config/yql_setting.h +++ b/yql/essentials/providers/common/config/yql_setting.h @@ -12,8 +12,17 @@ namespace NCommon { const TString ALL_CLUSTERS = "$all"; -// TODO: replace RUNTIME/PERCLUSTER with enum +enum class EConfSettingType { + Static, + StaticPerCluster, + Dynamic, +}; + +#ifdef YQL_BETTER_CONF_SETTING_API +template <typename TType, EConfSettingType SettingType = EConfSettingType::Dynamic> +#else template <typename TType, bool RUNTIME = true, bool PERCLUSTER = false> +#endif class TConfSetting { public: TConfSetting() = default; @@ -25,7 +34,11 @@ public: ~TConfSetting() = default; bool IsRuntime() const { +#ifdef YQL_BETTER_CONF_SETTING_API + return SettingType == EConfSettingType::Dynamic; +#else return RUNTIME; +#endif } TType& operator[](const TString& cluster) { @@ -81,7 +94,11 @@ private: }; template <typename TType> +#ifdef YQL_BETTER_CONF_SETTING_API +class TConfSetting<TType, EConfSettingType::Static> { +#else class TConfSetting<TType, false, false> { +#endif public: TConfSetting() = default; TConfSetting(const TType& value) diff --git a/yt/yql/providers/yt/common/yql_yt_settings.h b/yt/yql/providers/yt/common/yql_yt_settings.h index 386bb651c6b..5cde259c974 100644 --- a/yt/yql/providers/yt/common/yql_yt_settings.h +++ b/yt/yql/providers/yt/common/yql_yt_settings.h @@ -83,248 +83,268 @@ enum class ERuntimeClusterSelectionMode { }; struct TYtSettings { +private: +#ifdef YQL_BETTER_CONF_SETTING_API + static constexpr NCommon::EConfSettingType Static = NCommon::EConfSettingType::Static; + static constexpr NCommon::EConfSettingType Dynamic = NCommon::EConfSettingType::Dynamic; + static constexpr NCommon::EConfSettingType StaticPerCluster = NCommon::EConfSettingType::StaticPerCluster; +#else + static constexpr bool Static = false; + static constexpr bool Dynamic = true; +#endif +public: + using TConstPtr = std::shared_ptr<const TYtSettings>; // should be static, because are used on earlier stages // static per-cluster +#ifdef YQL_BETTER_CONF_SETTING_API + NCommon::TConfSetting<TGUID, StaticPerCluster> ExternalTx; + NCommon::TConfSetting<TString, StaticPerCluster> TmpFolder; + NCommon::TConfSetting<TString, StaticPerCluster> TablesTmpFolder; + NCommon::TConfSetting<TString, StaticPerCluster> BinaryTmpFolder; + NCommon::TConfSetting<TString, StaticPerCluster> StaticPool; + NCommon::TConfSetting<TString, StaticPerCluster> CoreDumpPath; +#else NCommon::TConfSetting<TGUID, false, true> ExternalTx; NCommon::TConfSetting<TString, false, true> TmpFolder; NCommon::TConfSetting<TString, false, true> TablesTmpFolder; NCommon::TConfSetting<TString, false, true> BinaryTmpFolder; NCommon::TConfSetting<TString, false, true> StaticPool; NCommon::TConfSetting<TString, false, true> CoreDumpPath; +#endif // static global - NCommon::TConfSetting<TString, false> Auth; - NCommon::TConfSetting<TDuration, false> TempTablesTtl; - NCommon::TConfSetting<bool, false> KeepTempTables; - NCommon::TConfSetting<ui32, false> InflightTempTablesLimit; - NCommon::TConfSetting<EReleaseTempDataMode, false> ReleaseTempData; - NCommon::TConfSetting<bool, false> IgnoreYamrDsv; - NCommon::TConfSetting<bool, false> IgnoreWeakSchema; - NCommon::TConfSetting<ui32, false> InferSchema; - NCommon::TConfSetting<ui32, false> ForceInferSchema; - NCommon::TConfSetting<ui32, false> InferSchemaTableCountThreshold; - NCommon::TConfSetting<NSize::TSize, false> DefaultCalcMemoryLimit; - NCommon::TConfSetting<ui32, false> ParallelOperationsLimit; - NCommon::TConfSetting<ui32, false> LocalCalcLimit; - NCommon::TConfSetting<EQueryCacheMode, false> QueryCacheMode; - NCommon::TConfSetting<bool, false> QueryCacheIgnoreTableRevision; - NCommon::TConfSetting<TString, false> QueryCacheSalt; - NCommon::TConfSetting<TDuration, false> QueryCacheTtl; - NCommon::TConfSetting<bool, false> QueryCacheUseExpirationTimeout; - NCommon::TConfSetting<bool, false> QueryCacheUseForCalc; - NCommon::TConfSetting<ui32, false> DefaultMaxJobFails; - NCommon::TConfSetting<TString, false> DefaultCluster; - NCommon::TConfSetting<TDuration, false> BinaryExpirationInterval; - NCommon::TConfSetting<bool, false> IgnoreTypeV3; - NCommon::TConfSetting<bool, false> _UseMultisetAttributes; - NCommon::TConfSetting<TDuration, false> FileCacheTtl; - NCommon::TConfSetting<TString, false> _ImpersonationUser; - NCommon::TConfSetting<EInferSchemaMode, false> InferSchemaMode; - NCommon::TConfSetting<ui32, false> BatchListFolderConcurrency; - NCommon::TConfSetting<bool, false> ForceTmpSecurity; - NCommon::TConfSetting<ERuntimeClusterSelectionMode, false> RuntimeClusterSelection; - NCommon::TConfSetting<TString, false> DefaultRuntimeCluster; - NCommon::TConfSetting<bool, false> _ForbidSensitiveDataInOperationSpec; - NCommon::TConfSetting<NSize::TSize, false> _LocalTableContentLimit; - NCommon::TConfSetting<bool, false> EnableDynamicStoreReadInDQ; + NCommon::TConfSetting<TString, Static> Auth; + NCommon::TConfSetting<TDuration, Static> TempTablesTtl; + NCommon::TConfSetting<bool, Static> KeepTempTables; + NCommon::TConfSetting<ui32, Static> InflightTempTablesLimit; + NCommon::TConfSetting<EReleaseTempDataMode, Static> ReleaseTempData; + NCommon::TConfSetting<bool, Static> IgnoreYamrDsv; + NCommon::TConfSetting<bool, Static> IgnoreWeakSchema; + NCommon::TConfSetting<ui32, Static> InferSchema; + NCommon::TConfSetting<ui32, Static> ForceInferSchema; + NCommon::TConfSetting<ui32, Static> InferSchemaTableCountThreshold; + NCommon::TConfSetting<NSize::TSize, Static> DefaultCalcMemoryLimit; + NCommon::TConfSetting<ui32, Static> ParallelOperationsLimit; + NCommon::TConfSetting<ui32, Static> LocalCalcLimit; + NCommon::TConfSetting<EQueryCacheMode, Static> QueryCacheMode; + NCommon::TConfSetting<bool, Static> QueryCacheIgnoreTableRevision; + NCommon::TConfSetting<TString, Static> QueryCacheSalt; + NCommon::TConfSetting<TDuration, Static> QueryCacheTtl; + NCommon::TConfSetting<bool, Static> QueryCacheUseExpirationTimeout; + NCommon::TConfSetting<bool, Static> QueryCacheUseForCalc; + NCommon::TConfSetting<ui32, Static> DefaultMaxJobFails; + NCommon::TConfSetting<TString, Static> DefaultCluster; + NCommon::TConfSetting<TDuration, Static> BinaryExpirationInterval; + NCommon::TConfSetting<bool, Static> IgnoreTypeV3; + NCommon::TConfSetting<bool, Static> _UseMultisetAttributes; + NCommon::TConfSetting<TDuration, Static> FileCacheTtl; + NCommon::TConfSetting<TString, Static> _ImpersonationUser; + NCommon::TConfSetting<EInferSchemaMode, Static> InferSchemaMode; + NCommon::TConfSetting<ui32, Static> BatchListFolderConcurrency; + NCommon::TConfSetting<bool, Static> ForceTmpSecurity; + NCommon::TConfSetting<ERuntimeClusterSelectionMode, Static> RuntimeClusterSelection; + NCommon::TConfSetting<TString, Static> DefaultRuntimeCluster; + NCommon::TConfSetting<bool, Static> _ForbidSensitiveDataInOperationSpec; + NCommon::TConfSetting<NSize::TSize, Static> _LocalTableContentLimit; + NCommon::TConfSetting<bool, Static> EnableDynamicStoreReadInDQ; // Job runtime - NCommon::TConfSetting<TString, true> Pool; - NCommon::TConfSetting<NSize::TSize, true> DefaultMemoryLimit; - NCommon::TConfSetting<double, true> DefaultMemoryReserveFactor; - NCommon::TConfSetting<double, true> DefaultMemoryDigestLowerBound; - NCommon::TConfSetting<NSize::TSize, true> MaxRowWeight; - NCommon::TConfSetting<NSize::TSize, true> MaxKeyWeight; - NCommon::TConfSetting<ui32, true> BufferRowCount; - NCommon::TConfSetting<NSize::TSize, true> DataSizePerJob; - NCommon::TConfSetting<NSize::TSize, true> DataSizePerSortJob; - NCommon::TConfSetting<NSize::TSize, true> DataSizePerMapJob; - NCommon::TConfSetting<NSize::TSize, true> DataSizePerPartition; - NCommon::TConfSetting<TDuration, true> DefaultLocalityTimeout; - NCommon::TConfSetting<TDuration, true> MapLocalityTimeout; - NCommon::TConfSetting<TDuration, true> ReduceLocalityTimeout; - NCommon::TConfSetting<TDuration, true> SortLocalityTimeout; - NCommon::TConfSetting<NSize::TSize, true> MinLocalityInputDataWeight; - NCommon::TConfSetting<ui64, true> MaxJobCount; - NCommon::TConfSetting<ui64, true> UserSlots; - NCommon::TConfSetting<double, true> DefaultOperationWeight; - NCommon::TConfSetting<double, true> DefaultMapSelectivityFactor; - NCommon::TConfSetting<bool, true> NightlyCompress; - NCommon::TConfSetting<TString, true> PublishedCompressionCodec; - NCommon::TConfSetting<TString, true> TemporaryCompressionCodec; - NCommon::TConfSetting<NYT::EErasureCodecAttr, true> PublishedErasureCodec; - NCommon::TConfSetting<NYT::EErasureCodecAttr, true> TemporaryErasureCodec; - NCommon::TConfSetting<TDuration, true> ClientMapTimeout; // TODO: yt_native - NCommon::TConfSetting<bool, true> UseTmpfs; - NCommon::TConfSetting<bool, true> SuspendIfAccountLimitExceeded; - NCommon::TConfSetting<NSize::TSize, true> ExtraTmpfsSize; - NCommon::TConfSetting<NYT::EOptimizeForAttr, true> OptimizeFor; // {scan, lookup} - NCommon::TConfSetting<TInstant, true> ExpirationDeadline; - NCommon::TConfSetting<TDuration, true> ExpirationInterval; - NCommon::TConfSetting<double, true> ScriptCpu; - NCommon::TConfSetting<double, true> PythonCpu; - NCommon::TConfSetting<double, true> JavascriptCpu; - NCommon::TConfSetting<double, true> ErasureCodecCpu; - NCommon::TConfSetting<double, true> ErasureCodecCpuForDq; - NCommon::TConfSetting<TSet<TString>, true> Owners; - NCommon::TConfSetting<TSet<TString>, true> OperationReaders; - NCommon::TConfSetting<TString, true> SchedulingTag; - NCommon::TConfSetting<TString, true> SchedulingTagFilter; - NCommon::TConfSetting<TSet<TString>, true> PoolTrees; - NCommon::TConfSetting<TSet<TString>, true> TentativePoolTrees; - NCommon::TConfSetting<ui32, true> TentativeTreeEligibilitySampleJobCount; - NCommon::TConfSetting<double, true> TentativeTreeEligibilityMaxJobDurationRatio; - NCommon::TConfSetting<ui32, true> TentativeTreeEligibilityMinJobDuration; - NCommon::TConfSetting<bool, true> UseDefaultTentativePoolTrees; - NCommon::TConfSetting<TString, true> IntermediateAccount; - NCommon::TConfSetting<ui32, true> IntermediateReplicationFactor; - NCommon::TConfSetting<ui32, true> PublishedReplicationFactor; - NCommon::TConfSetting<ui32, true> TemporaryReplicationFactor; - NCommon::TConfSetting<TString, true> AutoMerge; // {relaxed, economy, disabled} - NCommon::TConfSetting<TString, true> PublishedAutoMerge; - NCommon::TConfSetting<TString, true> TemporaryAutoMerge; - NCommon::TConfSetting<TVector<TString>, true> LayerPaths; - NCommon::TConfSetting<TString, true> DockerImage; - NCommon::TConfSetting<NYT::TNode, true> JobEnv; - NCommon::TConfSetting<NYT::TNode, true> OperationSpec; - NCommon::TConfSetting<NYT::TNode, true> FmrOperationSpec; - NCommon::TConfSetting<NYT::TNode, true> Annotations; - NCommon::TConfSetting<NYT::TNode, true> StartedBy; - NCommon::TConfSetting<NYT::TNode, true> Description; - NCommon::TConfSetting<bool, true> UseSkiff; - NCommon::TConfSetting<ui32, true> TableContentCompressLevel; - NCommon::TConfSetting<bool, true> DisableJobSplitting; - NCommon::TConfSetting<EUseColumnarStatisticsMode, true> UseColumnarStatistics; - NCommon::TConfSetting<ETableContentDeliveryMode, true> TableContentDeliveryMode; - NCommon::TConfSetting<bool, true> TableContentUseSkiff; - NCommon::TConfSetting<TString, true> TableContentTmpFolder; - NCommon::TConfSetting<bool, true> TableContentColumnarStatistics; - NCommon::TConfSetting<TString, true> GeobaseDownloadUrl; - NCommon::TConfSetting<ui32, true> MaxSpeculativeJobCountPerTask; - NCommon::TConfSetting<NSize::TSize, true> LLVMMemSize; - NCommon::TConfSetting<NSize::TSize, true> LLVMPerNodeMemSize; - NCommon::TConfSetting<ui64, true> LLVMNodeCountLimit; - NCommon::TConfSetting<NSize::TSize, true> SamplingIoBlockSize; - NCommon::TConfSetting<NYT::TNode, true> PublishedMedia; - NCommon::TConfSetting<NYT::TNode, true> TemporaryMedia; - NCommon::TConfSetting<TString, true> PublishedPrimaryMedium; - NCommon::TConfSetting<TString, true> TemporaryPrimaryMedium; - NCommon::TConfSetting<TString, true> IntermediateDataMedium; - NCommon::TConfSetting<TString, true> PrimaryMedium; - NCommon::TConfSetting<ui64, true> QueryCacheChunkLimit; - NCommon::TConfSetting<ui64, true> NativeYtTypeCompatibility; - NCommon::TConfSetting<bool, true> _UseKeyBoundApi; - NCommon::TConfSetting<TString, true> NetworkProject; - NCommon::TConfSetting<bool, true> _EnableYtPartitioning; - NCommon::TConfSetting<bool, true> ForceJobSizeAdjuster; - NCommon::TConfSetting<bool, true> EnforceJobUtc; - NCommon::TConfSetting<bool, true> UseRPCReaderInDQ; - NCommon::TConfSetting<size_t, true> DQRPCReaderInflight; - NCommon::TConfSetting<TDuration, true> DQRPCReaderTimeout; - NCommon::TConfSetting<TSet<TString>, true> BlockReaderSupportedTypes; - NCommon::TConfSetting<TSet<NUdf::EDataSlot>, true> BlockReaderSupportedDataTypes; - NCommon::TConfSetting<TString, true> _BinaryCacheFolder; - NCommon::TConfSetting<TString, true> RuntimeCluster; - NCommon::TConfSetting<bool, true> _AllowRemoteClusterInput; - NCommon::TConfSetting<bool, true> _EnableDq; + NCommon::TConfSetting<TString, Dynamic> Pool; + NCommon::TConfSetting<NSize::TSize, Dynamic> DefaultMemoryLimit; + NCommon::TConfSetting<double, Dynamic> DefaultMemoryReserveFactor; + NCommon::TConfSetting<double, Dynamic> DefaultMemoryDigestLowerBound; + NCommon::TConfSetting<NSize::TSize, Dynamic> MaxRowWeight; + NCommon::TConfSetting<NSize::TSize, Dynamic> MaxKeyWeight; + NCommon::TConfSetting<ui32, Dynamic> BufferRowCount; + NCommon::TConfSetting<NSize::TSize, Dynamic> DataSizePerJob; + NCommon::TConfSetting<NSize::TSize, Dynamic> DataSizePerSortJob; + NCommon::TConfSetting<NSize::TSize, Dynamic> DataSizePerMapJob; + NCommon::TConfSetting<NSize::TSize, Dynamic> DataSizePerPartition; + NCommon::TConfSetting<TDuration, Dynamic> DefaultLocalityTimeout; + NCommon::TConfSetting<TDuration, Dynamic> MapLocalityTimeout; + NCommon::TConfSetting<TDuration, Dynamic> ReduceLocalityTimeout; + NCommon::TConfSetting<TDuration, Dynamic> SortLocalityTimeout; + NCommon::TConfSetting<NSize::TSize, Dynamic> MinLocalityInputDataWeight; + NCommon::TConfSetting<ui64, Dynamic> MaxJobCount; + NCommon::TConfSetting<ui64, Dynamic> UserSlots; + NCommon::TConfSetting<double, Dynamic> DefaultOperationWeight; + NCommon::TConfSetting<double, Dynamic> DefaultMapSelectivityFactor; + NCommon::TConfSetting<bool, Dynamic> NightlyCompress; + NCommon::TConfSetting<TString, Dynamic> PublishedCompressionCodec; + NCommon::TConfSetting<TString, Dynamic> TemporaryCompressionCodec; + NCommon::TConfSetting<NYT::EErasureCodecAttr, Dynamic> PublishedErasureCodec; + NCommon::TConfSetting<NYT::EErasureCodecAttr, Dynamic> TemporaryErasureCodec; + NCommon::TConfSetting<TDuration, Dynamic> ClientMapTimeout; // TODO: yt_native + NCommon::TConfSetting<bool, Dynamic> UseTmpfs; + NCommon::TConfSetting<bool, Dynamic> SuspendIfAccountLimitExceeded; + NCommon::TConfSetting<NSize::TSize, Dynamic> ExtraTmpfsSize; + NCommon::TConfSetting<NYT::EOptimizeForAttr, Dynamic> OptimizeFor; // {scan, lookup} + NCommon::TConfSetting<TInstant, Dynamic> ExpirationDeadline; + NCommon::TConfSetting<TDuration, Dynamic> ExpirationInterval; + NCommon::TConfSetting<double, Dynamic> ScriptCpu; + NCommon::TConfSetting<double, Dynamic> PythonCpu; + NCommon::TConfSetting<double, Dynamic> JavascriptCpu; + NCommon::TConfSetting<double, Dynamic> ErasureCodecCpu; + NCommon::TConfSetting<double, Dynamic> ErasureCodecCpuForDq; + NCommon::TConfSetting<TSet<TString>, Dynamic> Owners; + NCommon::TConfSetting<TSet<TString>, Dynamic> OperationReaders; + NCommon::TConfSetting<TString, Dynamic> SchedulingTag; + NCommon::TConfSetting<TString, Dynamic> SchedulingTagFilter; + NCommon::TConfSetting<TSet<TString>, Dynamic> PoolTrees; + NCommon::TConfSetting<TSet<TString>, Dynamic> TentativePoolTrees; + NCommon::TConfSetting<ui32, Dynamic> TentativeTreeEligibilitySampleJobCount; + NCommon::TConfSetting<double, Dynamic> TentativeTreeEligibilityMaxJobDurationRatio; + NCommon::TConfSetting<ui32, Dynamic> TentativeTreeEligibilityMinJobDuration; + NCommon::TConfSetting<bool, Dynamic> UseDefaultTentativePoolTrees; + NCommon::TConfSetting<TString, Dynamic> IntermediateAccount; + NCommon::TConfSetting<ui32, Dynamic> IntermediateReplicationFactor; + NCommon::TConfSetting<ui32, Dynamic> PublishedReplicationFactor; + NCommon::TConfSetting<ui32, Dynamic> TemporaryReplicationFactor; + NCommon::TConfSetting<TString, Dynamic> AutoMerge; // {relaxed, economy, disabled} + NCommon::TConfSetting<TString, Dynamic> PublishedAutoMerge; + NCommon::TConfSetting<TString, Dynamic> TemporaryAutoMerge; + NCommon::TConfSetting<TVector<TString>, Dynamic> LayerPaths; + NCommon::TConfSetting<TString, Dynamic> DockerImage; + NCommon::TConfSetting<NYT::TNode, Dynamic> JobEnv; + NCommon::TConfSetting<NYT::TNode, Dynamic> OperationSpec; + NCommon::TConfSetting<NYT::TNode, Dynamic> FmrOperationSpec; + NCommon::TConfSetting<NYT::TNode, Dynamic> Annotations; + NCommon::TConfSetting<NYT::TNode, Dynamic> StartedBy; + NCommon::TConfSetting<NYT::TNode, Dynamic> Description; + NCommon::TConfSetting<bool, Dynamic> UseSkiff; + NCommon::TConfSetting<ui32, Dynamic> TableContentCompressLevel; + NCommon::TConfSetting<bool, Dynamic> DisableJobSplitting; + NCommon::TConfSetting<EUseColumnarStatisticsMode, Dynamic> UseColumnarStatistics; + NCommon::TConfSetting<ETableContentDeliveryMode, Dynamic> TableContentDeliveryMode; + NCommon::TConfSetting<bool, Dynamic> TableContentUseSkiff; + NCommon::TConfSetting<TString, Dynamic> TableContentTmpFolder; + NCommon::TConfSetting<bool, Dynamic> TableContentColumnarStatistics; + NCommon::TConfSetting<TString, Dynamic> GeobaseDownloadUrl; + NCommon::TConfSetting<ui32, Dynamic> MaxSpeculativeJobCountPerTask; + NCommon::TConfSetting<NSize::TSize, Dynamic> LLVMMemSize; + NCommon::TConfSetting<NSize::TSize, Dynamic> LLVMPerNodeMemSize; + NCommon::TConfSetting<ui64, Dynamic> LLVMNodeCountLimit; + NCommon::TConfSetting<NSize::TSize, Dynamic> SamplingIoBlockSize; + NCommon::TConfSetting<NYT::TNode, Dynamic> PublishedMedia; + NCommon::TConfSetting<NYT::TNode, Dynamic> TemporaryMedia; + NCommon::TConfSetting<TString, Dynamic> PublishedPrimaryMedium; + NCommon::TConfSetting<TString, Dynamic> TemporaryPrimaryMedium; + NCommon::TConfSetting<TString, Dynamic> IntermediateDataMedium; + NCommon::TConfSetting<TString, Dynamic> PrimaryMedium; + NCommon::TConfSetting<ui64, Dynamic> QueryCacheChunkLimit; + NCommon::TConfSetting<ui64, Dynamic> NativeYtTypeCompatibility; + NCommon::TConfSetting<bool, Dynamic> _UseKeyBoundApi; + NCommon::TConfSetting<TString, Dynamic> NetworkProject; + NCommon::TConfSetting<bool, Dynamic> _EnableYtPartitioning; + NCommon::TConfSetting<bool, Dynamic> ForceJobSizeAdjuster; + NCommon::TConfSetting<bool, Dynamic> EnforceJobUtc; + NCommon::TConfSetting<bool, Dynamic> UseRPCReaderInDQ; + NCommon::TConfSetting<size_t, Dynamic> DQRPCReaderInflight; + NCommon::TConfSetting<TDuration, Dynamic> DQRPCReaderTimeout; + NCommon::TConfSetting<TSet<TString>, Dynamic> BlockReaderSupportedTypes; + NCommon::TConfSetting<TSet<NUdf::EDataSlot>, Dynamic> BlockReaderSupportedDataTypes; + NCommon::TConfSetting<TString, Dynamic> _BinaryCacheFolder; + NCommon::TConfSetting<TString, Dynamic> RuntimeCluster; + NCommon::TConfSetting<bool, Dynamic> _AllowRemoteClusterInput; + NCommon::TConfSetting<bool, Dynamic> _EnableDq; // Optimizers (static global) - NCommon::TConfSetting<ui32, false> ExtendTableLimit; // Deprecated. Use MaxInputTables instead - NCommon::TConfSetting<NSize::TSize, false> CommonJoinCoreLimit; - NCommon::TConfSetting<NSize::TSize, false> CombineCoreLimit; - NCommon::TConfSetting<NSize::TSize, false> SwitchLimit; - NCommon::TConfSetting<ui64, false> JoinMergeTablesLimit; - NCommon::TConfSetting<bool, false> JoinMergeUseSmallAsPrimary; - NCommon::TConfSetting<NSize::TSize, false> JoinMergeReduceJobMaxSize; - NCommon::TConfSetting<double, false> JoinMergeUnsortedFactor; // (>=0.0) - NCommon::TConfSetting<bool, false> JoinMergeForce; - NCommon::TConfSetting<NSize::TSize, false> MapJoinLimit; - NCommon::TConfSetting<ui64, false> MapJoinShardMinRows; - NCommon::TConfSetting<ui64, false> MapJoinShardCount; // [1-10] - NCommon::TConfSetting<bool, false> MapJoinUseFlow; - NCommon::TConfSetting<bool, false> BlockMapJoin; - NCommon::TConfSetting<NSize::TSize, false> LookupJoinLimit; - NCommon::TConfSetting<ui64, false> LookupJoinMaxRows; - NCommon::TConfSetting<NSize::TSize, false> EvaluationTableSizeLimit; - NCommon::TConfSetting<TSet<TString>, false> DisableOptimizers; - NCommon::TConfSetting<ui32, false> MaxInputTables; - NCommon::TConfSetting<ui32, false> MaxInputTablesForSortedMerge; - NCommon::TConfSetting<ui32, false> MaxOutputTables; - NCommon::TConfSetting<bool, false> DisableFuseOperations; - NCommon::TConfSetting<bool, false> EnableFuseMapToMapReduce; - NCommon::TConfSetting<NSize::TSize, false> MaxExtraJobMemoryToFuseOperations; - NCommon::TConfSetting<double, false> MaxReplicationFactorToFuseOperations; - NCommon::TConfSetting<ui32, false> MaxOperationFiles; - NCommon::TConfSetting<NSize::TSize, false> MinPublishedAvgChunkSize; - NCommon::TConfSetting<NSize::TSize, false> MinTempAvgChunkSize; - NCommon::TConfSetting<ui32, false> TopSortMaxLimit; - NCommon::TConfSetting<NSize::TSize, false> TopSortSizePerJob; - NCommon::TConfSetting<ui32, false> TopSortRowMultiplierPerJob; - NCommon::TConfSetting<bool, false> JoinUseColumnarStatistics; // Deprecated. Use JoinCollectColumnarStatistics instead - NCommon::TConfSetting<EJoinCollectColumnarStatisticsMode, false> JoinCollectColumnarStatistics; - NCommon::TConfSetting<NYT::EColumnarStatisticsFetcherMode, false> JoinColumnarStatisticsFetcherMode; - NCommon::TConfSetting<bool, false> JoinWaitAllInputs; - NCommon::TConfSetting<bool, false> JoinAllowColumnRenames; - NCommon::TConfSetting<bool, false> JoinMergeSetTopLevelFullSort; - NCommon::TConfSetting<bool, false> JoinEnableStarJoin; - NCommon::TConfSetting<NSize::TSize, false> FolderInlineDataLimit; - NCommon::TConfSetting<ui32, false> FolderInlineItemsLimit; - NCommon::TConfSetting<NSize::TSize, false> TableContentMinAvgChunkSize; - NCommon::TConfSetting<ui32, false> TableContentMaxInputTables; - NCommon::TConfSetting<ui32, false> TableContentMaxChunksForNativeDelivery; - NCommon::TConfSetting<NSize::TSize, false> TableContentLocalExecution; - NCommon::TConfSetting<bool, false> UseTypeV2; - NCommon::TConfSetting<bool, false> UseNativeYtTypes; - NCommon::TConfSetting<bool, false> UseNativeDescSort; - NCommon::TConfSetting<bool, false> UseIntermediateSchema; - NCommon::TConfSetting<bool, false> UseIntermediateStreams; - NCommon::TConfSetting<bool, false> UseFlow; - NCommon::TConfSetting<ui16, false> WideFlowLimit; - NCommon::TConfSetting<bool, false> UseSystemColumns; - NCommon::TConfSetting<bool, false> HybridDqExecution; - NCommon::TConfSetting<TDuration, false> HybridDqTimeSpentLimit; - NCommon::TConfSetting<NSize::TSize, false> HybridDqDataSizeLimitForOrdered; - NCommon::TConfSetting<NSize::TSize, false> HybridDqDataSizeLimitForUnordered; - NCommon::TConfSetting<bool, false> HybridDqExecutionFallback; - NCommon::TConfSetting<bool, false> UseYqlRowSpecCompactForm; - NCommon::TConfSetting<bool, false> UseNewPredicateExtraction; - NCommon::TConfSetting<bool, false> PruneKeyFilterLambda; - NCommon::TConfSetting<bool, false> DqPruneKeyFilterLambda; - NCommon::TConfSetting<bool, false> UseQLFilter; - NCommon::TConfSetting<bool, false> PruneQLFilterLambda; - NCommon::TConfSetting<bool, false> MergeAdjacentPointRanges; - NCommon::TConfSetting<bool, false> KeyFilterForStartsWith; - NCommon::TConfSetting<ui64, false> MaxKeyRangeCount; - NCommon::TConfSetting<ui64, false> MaxChunksForDqRead; - NCommon::TConfSetting<bool, false> JoinCommonUseMapMultiOut; - NCommon::TConfSetting<bool, false> UseAggPhases; - NCommon::TConfSetting<bool, false> UsePartitionsByKeysForFinalAgg; - NCommon::TConfSetting<double, false> MaxCpuUsageToFuseMultiOuts; - NCommon::TConfSetting<double, false> MaxReplicationFactorToFuseMultiOuts; - NCommon::TConfSetting<ui64, false> ApplyStoredConstraints; - NCommon::TConfSetting<bool, false> ViewIsolation; - NCommon::TConfSetting<bool, false> PartitionByConstantKeysViaMap; - NCommon::TConfSetting<EColumnGroupMode, false> ColumnGroupMode; - NCommon::TConfSetting<ui16, false> MinColumnGroupSize; - NCommon::TConfSetting<ui16, false> MaxColumnGroups; - NCommon::TConfSetting<ui64, false> ExtendedStatsMaxChunkCount; - NCommon::TConfSetting<bool, false> JobBlockInput; - NCommon::TConfSetting<bool, false> JobBlockTableContent; - NCommon::TConfSetting<TSet<TString>, false> JobBlockInputSupportedTypes; - NCommon::TConfSetting<TSet<NUdf::EDataSlot>, false> JobBlockInputSupportedDataTypes; - NCommon::TConfSetting<EBlockOutputMode, false> JobBlockOutput; - NCommon::TConfSetting<TSet<TString>, false> JobBlockOutputSupportedTypes; - NCommon::TConfSetting<TSet<NUdf::EDataSlot>, false> JobBlockOutputSupportedDataTypes; - NCommon::TConfSetting<bool, false> _EnableYtDqProcessWriteConstraints; - NCommon::TConfSetting<bool, false> CompactForDistinct; - NCommon::TConfSetting<bool, false> DropUnusedKeysFromKeyFilter; - NCommon::TConfSetting<bool, false> ReportEquiJoinStats; - NCommon::TConfSetting<bool, false> UseColumnGroupsFromInputTables; - NCommon::TConfSetting<bool, false> UseNativeDynamicTableRead; - NCommon::TConfSetting<bool, false> DontForceTransformForInputTables; + NCommon::TConfSetting<ui32, Static> ExtendTableLimit; // Deprecated. Use MaxInputTables instead + NCommon::TConfSetting<NSize::TSize, Static> CommonJoinCoreLimit; + NCommon::TConfSetting<NSize::TSize, Static> CombineCoreLimit; + NCommon::TConfSetting<NSize::TSize, Static> SwitchLimit; + NCommon::TConfSetting<ui64, Static> JoinMergeTablesLimit; + NCommon::TConfSetting<bool, Static> JoinMergeUseSmallAsPrimary; + NCommon::TConfSetting<NSize::TSize, Static> JoinMergeReduceJobMaxSize; + NCommon::TConfSetting<double, Static> JoinMergeUnsortedFactor; // (>=0.0) + NCommon::TConfSetting<bool, Static> JoinMergeForce; + NCommon::TConfSetting<NSize::TSize, Static> MapJoinLimit; + NCommon::TConfSetting<ui64, Static> MapJoinShardMinRows; + NCommon::TConfSetting<ui64, Static> MapJoinShardCount; // [1-10] + NCommon::TConfSetting<bool, Static> MapJoinUseFlow; + NCommon::TConfSetting<bool, Static> BlockMapJoin; + NCommon::TConfSetting<NSize::TSize, Static> LookupJoinLimit; + NCommon::TConfSetting<ui64, Static> LookupJoinMaxRows; + NCommon::TConfSetting<NSize::TSize, Static> EvaluationTableSizeLimit; + NCommon::TConfSetting<TSet<TString>, Static> DisableOptimizers; + NCommon::TConfSetting<ui32, Static> MaxInputTables; + NCommon::TConfSetting<ui32, Static> MaxInputTablesForSortedMerge; + NCommon::TConfSetting<ui32, Static> MaxOutputTables; + NCommon::TConfSetting<bool, Static> DisableFuseOperations; + NCommon::TConfSetting<bool, Static> EnableFuseMapToMapReduce; + NCommon::TConfSetting<NSize::TSize, Static> MaxExtraJobMemoryToFuseOperations; + NCommon::TConfSetting<double, Static> MaxReplicationFactorToFuseOperations; + NCommon::TConfSetting<ui32, Static> MaxOperationFiles; + NCommon::TConfSetting<NSize::TSize, Static> MinPublishedAvgChunkSize; + NCommon::TConfSetting<NSize::TSize, Static> MinTempAvgChunkSize; + NCommon::TConfSetting<ui32, Static> TopSortMaxLimit; + NCommon::TConfSetting<NSize::TSize, Static> TopSortSizePerJob; + NCommon::TConfSetting<ui32, Static> TopSortRowMultiplierPerJob; + NCommon::TConfSetting<bool, Static> JoinUseColumnarStatistics; // Deprecated. Use JoinCollectColumnarStatistics instead + NCommon::TConfSetting<EJoinCollectColumnarStatisticsMode, Static> JoinCollectColumnarStatistics; + NCommon::TConfSetting<NYT::EColumnarStatisticsFetcherMode, Static> JoinColumnarStatisticsFetcherMode; + NCommon::TConfSetting<bool, Static> JoinWaitAllInputs; + NCommon::TConfSetting<bool, Static> JoinAllowColumnRenames; + NCommon::TConfSetting<bool, Static> JoinMergeSetTopLevelFullSort; + NCommon::TConfSetting<bool, Static> JoinEnableStarJoin; + NCommon::TConfSetting<NSize::TSize, Static> FolderInlineDataLimit; + NCommon::TConfSetting<ui32, Static> FolderInlineItemsLimit; + NCommon::TConfSetting<NSize::TSize, Static> TableContentMinAvgChunkSize; + NCommon::TConfSetting<ui32, Static> TableContentMaxInputTables; + NCommon::TConfSetting<ui32, Static> TableContentMaxChunksForNativeDelivery; + NCommon::TConfSetting<NSize::TSize, Static> TableContentLocalExecution; + NCommon::TConfSetting<bool, Static> UseTypeV2; + NCommon::TConfSetting<bool, Static> UseNativeYtTypes; + NCommon::TConfSetting<bool, Static> UseNativeDescSort; + NCommon::TConfSetting<bool, Static> UseIntermediateSchema; + NCommon::TConfSetting<bool, Static> UseIntermediateStreams; + NCommon::TConfSetting<bool, Static> UseFlow; + NCommon::TConfSetting<ui16, Static> WideFlowLimit; + NCommon::TConfSetting<bool, Static> UseSystemColumns; + NCommon::TConfSetting<bool, Static> HybridDqExecution; + NCommon::TConfSetting<TDuration, Static> HybridDqTimeSpentLimit; + NCommon::TConfSetting<NSize::TSize, Static> HybridDqDataSizeLimitForOrdered; + NCommon::TConfSetting<NSize::TSize, Static> HybridDqDataSizeLimitForUnordered; + NCommon::TConfSetting<bool, Static> HybridDqExecutionFallback; + NCommon::TConfSetting<bool, Static> UseYqlRowSpecCompactForm; + NCommon::TConfSetting<bool, Static> UseNewPredicateExtraction; + NCommon::TConfSetting<bool, Static> PruneKeyFilterLambda; + NCommon::TConfSetting<bool, Static> DqPruneKeyFilterLambda; + NCommon::TConfSetting<bool, Static> UseQLFilter; + NCommon::TConfSetting<bool, Static> PruneQLFilterLambda; + NCommon::TConfSetting<bool, Static> MergeAdjacentPointRanges; + NCommon::TConfSetting<bool, Static> KeyFilterForStartsWith; + NCommon::TConfSetting<ui64, Static> MaxKeyRangeCount; + NCommon::TConfSetting<ui64, Static> MaxChunksForDqRead; + NCommon::TConfSetting<bool, Static> JoinCommonUseMapMultiOut; + NCommon::TConfSetting<bool, Static> UseAggPhases; + NCommon::TConfSetting<bool, Static> UsePartitionsByKeysForFinalAgg; + NCommon::TConfSetting<double, Static> MaxCpuUsageToFuseMultiOuts; + NCommon::TConfSetting<double, Static> MaxReplicationFactorToFuseMultiOuts; + NCommon::TConfSetting<ui64, Static> ApplyStoredConstraints; + NCommon::TConfSetting<bool, Static> ViewIsolation; + NCommon::TConfSetting<bool, Static> PartitionByConstantKeysViaMap; + NCommon::TConfSetting<EColumnGroupMode, Static> ColumnGroupMode; + NCommon::TConfSetting<ui16, Static> MinColumnGroupSize; + NCommon::TConfSetting<ui16, Static> MaxColumnGroups; + NCommon::TConfSetting<ui64, Static> ExtendedStatsMaxChunkCount; + NCommon::TConfSetting<bool, Static> JobBlockInput; + NCommon::TConfSetting<bool, Static> JobBlockTableContent; + NCommon::TConfSetting<TSet<TString>, Static> JobBlockInputSupportedTypes; + NCommon::TConfSetting<TSet<NUdf::EDataSlot>, Static> JobBlockInputSupportedDataTypes; + NCommon::TConfSetting<EBlockOutputMode, Static> JobBlockOutput; + NCommon::TConfSetting<TSet<TString>, Static> JobBlockOutputSupportedTypes; + NCommon::TConfSetting<TSet<NUdf::EDataSlot>, Static> JobBlockOutputSupportedDataTypes; + NCommon::TConfSetting<bool, Static> _EnableYtDqProcessWriteConstraints; + NCommon::TConfSetting<bool, Static> CompactForDistinct; + NCommon::TConfSetting<bool, Static> DropUnusedKeysFromKeyFilter; + NCommon::TConfSetting<bool, Static> ReportEquiJoinStats; + NCommon::TConfSetting<bool, Static> UseColumnGroupsFromInputTables; + NCommon::TConfSetting<bool, Static> UseNativeDynamicTableRead; + NCommon::TConfSetting<bool, Static> DontForceTransformForInputTables; }; EReleaseTempDataMode GetReleaseTempDataMode(const TYtSettings& settings); |
