Add separate feature flag for export and import object filtering (#44983)

Co-authored-by: Cursor Agent <[email protected]>
Co-authored-by: st-shchetinin <[email protected]>
Co-authored-by: stanislav_shchetinin <[email protected]>
Co-authored-by: Ilnaz Nizametdinov <[email protected]>

10 files changed, 153 insertions, 35 deletions

diff --git a/ydb/core/backup/common/feature_flags.cpp b/ydb/core/backup/common/feature_flags.cpp
new file mode 100644
index 00000000000..f022c4ec809
--- /dev/null
+++ b/ydb/core/backup/common/feature_flags.cpp
@@ -0,0 +1,17 @@
+#include "feature_flags.h"
+
+#include <ydb/core/base/appdata.h>
+
+namespace NKikimr::NBackup {
+
+bool IsExportFilteringEnabled(const TAppData& appData) {
+    // Backward compatibility: clusters with only EnableEncryptedExport keep filtering API enabled.
+    return appData.FeatureFlags.GetEnableExportFiltering()
+        || appData.FeatureFlags.GetEnableEncryptedExport();
+}
+
+bool IsEncryptedExportEnabled(const TAppData& appData) {
+    return appData.FeatureFlags.GetEnableEncryptedExport();
+}
+
+} // namespace NKikimr::NBackup
diff --git a/ydb/core/backup/common/feature_flags.h b/ydb/core/backup/common/feature_flags.h
new file mode 100644
index 00000000000..f1d359fd529
--- /dev/null
+++ b/ydb/core/backup/common/feature_flags.h
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <ydb/core/base/appdata_fwd.h>
+
+namespace NKikimr::NBackup {
+
+bool IsExportFilteringEnabled(const TAppData& appData);
+
+bool IsEncryptedExportEnabled(const TAppData& appData);
+
+} // namespace NKikimr::NBackup
diff --git a/ydb/core/backup/common/ya.make b/ydb/core/backup/common/ya.make
index e5a59cc5e21..071d2cfdcfd 100644
--- a/ydb/core/backup/common/ya.make
+++ b/ydb/core/backup/common/ya.make
@@ -3,6 +3,7 @@ LIBRARY()
 SRCS(
     checksum.cpp
     encryption.cpp
+    feature_flags.cpp
     fields_wrappers.cpp
     metadata.cpp
 )
diff --git a/ydb/core/grpc_services/rpc_export.cpp b/ydb/core/grpc_services/rpc_export.cpp
index 0d25487d218..a5275e00c8a 100644
--- a/ydb/core/grpc_services/rpc_export.cpp
+++ b/ydb/core/grpc_services/rpc_export.cpp
@@ -7,6 +7,7 @@
 
 #include <ydb/public/api/protos/ydb_export.pb.h>
 #include <ydb/core/backup/common/encryption.h>
+#include <ydb/core/backup/common/feature_flags.h>
 #include <ydb/core/backup/regexp/regexp.h>
 #include <ydb/core/base/path.h>
 #include <ydb/core/tx/schemeshard/schemeshard_export.h>
@@ -570,6 +571,21 @@ public:
         const auto& settings = request.settings();
         InitCommonSourcePath();
 
+        if constexpr (TTraits::HasEncryption) {
+            if (settings.has_encryption_settings()) { // Validate that it is possible to encrypt with these settings
+                if (!NBackup::IsEncryptedExportEnabled(*AppData())) {
+                    return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Export encryption is not supported in current configuration");
+                }
+                if (!TTraits::HasDestination(settings)) {
+                    return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "No destination prefix specified for encrypted export");
+                }
+
+                if (!ValidateEncryptionParameters()) {
+                    return;
+                }
+            }
+        }
+
         try {
             ExcludeRegexps = NBackup::CombineRegexps(settings.exclude_regexps());
         } catch (const std::exception& ex) {
@@ -602,9 +618,8 @@ public:
                 return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Items are not set");
             }
         } else {
-            const bool encryptedExportFeatureFlag = AppData()->FeatureFlags.GetEnableEncryptedExport();
             const bool commonDestSpecified = TTraits::HasDestination(settings);
-            if (!encryptedExportFeatureFlag) {
+            if (!NBackup::IsExportFilteringEnabled(*AppData())) {
                 // Check that no new fields are specified
                 if constexpr (IsS3Export) {
                     if (commonDestSpecified) {
@@ -614,9 +629,6 @@ public:
                 if (!settings.source_path().empty()) {
                     return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Source path is not supported in current configuration");
                 }
-                if (settings.has_encryption_settings()) {
-                    return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Export encryption is not supported in current configuration");
-                }
                 if constexpr (IsFsExport) {
                     if (settings.items().empty()) {
                         return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR,
@@ -639,17 +651,6 @@ public:
                 }
             }
         }
-        if constexpr (TTraits::HasEncryption) {
-            if (settings.has_encryption_settings()) { // Validate that it is possible to encrypt with these settings
-                if (!TTraits::HasDestination(settings)) {
-                    return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "No destination prefix specified for encrypted export");
-                }
-
-                if (!ValidateEncryptionParameters()) {
-                    return;
-                }
-            }
-        }
 
         if constexpr (TTraits::HasCompression) {
             if (settings.compression()) {
diff --git a/ydb/core/grpc_services/rpc_import.cpp b/ydb/core/grpc_services/rpc_import.cpp
index e0c091efa6a..ae432c4a884 100644
--- a/ydb/core/grpc_services/rpc_import.cpp
+++ b/ydb/core/grpc_services/rpc_import.cpp
@@ -7,6 +7,7 @@
 
 #include <ydb/public/api/protos/ydb_import.pb.h>
 
+#include <ydb/core/backup/common/feature_flags.h>
 #include <ydb/core/backup/regexp/regexp.h>
 #include <ydb/core/tx/schemeshard/schemeshard_import.h>
 
@@ -127,6 +128,8 @@ public:
         }
 
         const auto& settings = request.settings();
+        const bool exportFilteringEnabled = NBackup::IsExportFilteringEnabled(*AppData());
+        const bool encryptedExportEnabled = NBackup::IsEncryptedExportEnabled(*AppData());
         try {
             // Validate regexps
             NBackup::CombineRegexps(settings.exclude_regexps());
@@ -134,26 +137,34 @@ public:
             return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, TStringBuilder() << "Invalid regexp: " << ex.what());
         }
 
-        const bool encryptedExportFeatureFlag = AppData()->FeatureFlags.GetEnableEncryptedExport();
         const bool commonSourcePathSpecified = !TTraits::GetCommonSourcePath(settings).empty();
         if constexpr (IsS3Import) {
-            if (!encryptedExportFeatureFlag) {
+            if (!exportFilteringEnabled) {
                 if (commonSourcePathSpecified) {
                     return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Source prefix is not supported in current configuration");
                 }
             }
         }
-        if (!encryptedExportFeatureFlag) {
+        if (!exportFilteringEnabled) {
             if (!settings.destination_path().empty()) {
                 return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Destination path is not supported in current configuration");
             }
-            if (settings.has_encryption_settings()) {
+        }
+        if (settings.has_encryption_settings()) {
+            if (!encryptedExportEnabled) {
                 return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Export encryption is not supported in current configuration");
             }
+            if (settings.encryption_settings().symmetric_key().key().empty()) {
+                return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "No encryption key specified");
+            }
+            if (!commonSourcePathSpecified) {
+                return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "No source prefix specified");
+            }
         }
+
         if (settings.items().empty() && !commonSourcePathSpecified) {
             return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "No source prefix specified. Don't know where to import from");
-        } else if (settings.items().empty() && !encryptedExportFeatureFlag) {
+        } else if (settings.items().empty() && !exportFilteringEnabled) {
             return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "No items to import. Don't know where to import from");
         }
         for (const auto& item : settings.items()) {
@@ -163,7 +174,7 @@ public:
             }
             if constexpr (IsS3Import) {
                 if (!item.source_path().empty()) {
-                    if (!encryptedExportFeatureFlag) {
+                    if (!exportFilteringEnabled) {
                         return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Item source path is not supported in current configuration");
                     }
                     if (!commonSourcePathSpecified) {
@@ -171,18 +182,17 @@ public:
                     }
                 }
             }
+            if constexpr (IsFsImport) {
+                if (!item.source_path_db().empty()) {
+                    if (!exportFilteringEnabled) {
+                        return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Item source path is not supported in current configuration");
+                    }
+                }
+            }
             if (TTraits::IsEmptyItem(item)) {
                 return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "Empty import item was specified");
             }
         }
-        if (settings.has_encryption_settings()) {
-            if (settings.encryption_settings().symmetric_key().key().empty()) {
-                return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "No encryption key specified");
-            }
-            if (!commonSourcePathSpecified) {
-                return this->Reply(StatusIds::BAD_REQUEST, TIssuesIds::DEFAULT_ERROR, "No source prefix specified");
-            }
-        }
 
         if constexpr (IsFsImport) {
             if (!TFsPath(settings.base_path()).IsAbsolute()) {
diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto
index 301e84e5913..7f1d5b1374a 100644
--- a/ydb/core/protos/feature_flags.proto
+++ b/ydb/core/protos/feature_flags.proto
@@ -205,6 +205,7 @@ message TFeatureFlags {
     optional bool EnableLoginCache = 178 [default = false];
     optional bool SwitchToConfigV2 = 179 [default = false];
     optional bool SwitchToConfigV1 = 180 [default = false];
+    // Implies EnableExportFiltering.
     optional bool EnableEncryptedExport = 181 [default = false];
     optional bool EnableAlterDatabase = 182 [default = false];
     optional bool EnableExportAutoDropping = 183 [default = true];
@@ -333,4 +334,5 @@ message TFeatureFlags {
     // is used or auto-provisioned, and such indexes fall back to requiring a single integer primary key.
     optional bool EnableFulltextIndexRowId = 289 [default = true];
     optional bool EnableForcedColumnCompactions = 290 [default = false];
+    optional bool EnableExportFiltering = 291 [default = false];
 }
diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
index 4c732581954..5c94a218cd5 100644
--- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
@@ -12,6 +12,7 @@
 #include <ydb/public/api/protos/ydb_status_codes.pb.h>
 
 #include <ydb/core/backup/common/encryption.h>
+#include <ydb/core/backup/common/feature_flags.h>
 #include <ydb/core/backup/common/fields_wrappers.h>
 
 #include <util/generic/algorithm.h>
@@ -1483,8 +1484,8 @@ private:
         case EState::CreateExportDir: {
             exportInfo->WaitTxId = InvalidTxId;
 
-            const bool supportEncryptedExport = AppData()->FeatureFlags.GetEnableEncryptedExport();
-            if (TString issues; supportEncryptedExport && !FillExportMetadata(*exportInfo, issues)) {
+            const bool supportExportFiltering = NBackup::IsExportFilteringEnabled(*AppData());
+            if (TString issues; supportExportFiltering && !FillExportMetadata(*exportInfo, issues)) {
                 exportInfo->State = EState::Cancelled;
                 exportInfo->EndTime = TAppData::TimeProvider->Now();
                 exportInfo->Issue = issues;
@@ -1492,7 +1493,7 @@ private:
                 break;
             }
 
-            if (supportEncryptedExport && UploadExportMetadata(*exportInfo, ctx)) {
+            if (supportExportFiltering && UploadExportMetadata(*exportInfo, ctx)) {
                 exportInfo->State = EState::UploadExportMetadata;
 
                 // Persist modified metadata and new settings
diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp
index 3dc8d33d8e9..8a6260a0d46 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp
@@ -24,6 +24,8 @@
 #include <ydb/core/ydb_convert/table_description.h>
 #include <ydb/core/ydb_convert/ydb_convert.h>
 
+#include <ydb/core/backup/common/feature_flags.h>
+
 #include <util/generic/algorithm.h>
 #include <util/generic/maybe.h>
 #include <util/generic/ptr.h>
@@ -300,7 +302,7 @@ struct TSchemeShard::TImport::TTxCreate: public TSchemeShard::TXxport::TTxBase {
                     settings.set_scheme(Ydb::Import::ImportFromS3Settings::HTTPS);
                 }
 
-                if (!settings.source_prefix().empty() && AppData()->FeatureFlags.GetEnableEncryptedExport()) {
+                if (!settings.source_prefix().empty() && NBackup::IsExportFilteringEnabled(*AppData())) {
                     initialState = TImportInfo::EState::DownloadExportMetadata;
                 }
 
@@ -327,7 +329,7 @@ struct TSchemeShard::TImport::TTxCreate: public TSchemeShard::TXxport::TTxBase {
                     return Reply(std::move(response), Ydb::StatusIds::UNSUPPORTED, "The feature flag \"EnableFsBackups\" is disabled. The operation cannot be performed.");
                 }
 
-                if (AppData()->FeatureFlags.GetEnableEncryptedExport()) {
+                if (NBackup::IsExportFilteringEnabled(*AppData())) {
                     initialState = TImportInfo::EState::DownloadExportMetadata;
                 }
 
diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make
index 1442cd9c185..78d45debc4a 100644
--- a/ydb/core/tx/schemeshard/ya.make
+++ b/ydb/core/tx/schemeshard/ya.make
@@ -363,6 +363,7 @@ PEERDIR(
     ydb/core/actorlib_impl
     ydb/core/audit
     ydb/core/base
+    ydb/core/backup/common
     ydb/core/backup/regexp
     ydb/core/blob_depot
     ydb/core/blobstorage/base
diff --git a/ydb/services/ydb/backup_ut/backup_path_ut.cpp b/ydb/services/ydb/backup_ut/backup_path_ut.cpp
index 0297a6ff37a..ca900e85f26 100644
--- a/ydb/services/ydb/backup_ut/backup_path_ut.cpp
+++ b/ydb/services/ydb/backup_ut/backup_path_ut.cpp
@@ -607,6 +607,69 @@ void ExportWithExcludeRegexpsImpl(TBackupTestFixture& f, bool isOlap) {
 }
 
 template <typename TExportSettings, typename TBackupTestFixture>
+void ExportFilteringWithoutEncryptionImpl(TBackupTestFixture& f, bool isOlap) {
+    TBackupTraits<TExportSettings> traits;
+    const TString prefix = traits.FilePrefix();
+    f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableFsBackups(true);
+    f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableExportFiltering(true);
+    f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableEncryptedExport(false);
+
+    {
+        auto exportSettings = traits.MakeExportSettings(f, "/Root/RecursiveFolderProcessing");
+        exportSettings
+            .AppendExcludeRegexp("^Table$")
+            .AppendExcludeRegexp("^dir1$")
+            .AppendExcludeRegexp("^dir1/Table");
+        auto res = traits.Export(f, exportSettings);
+        f.WaitOpSuccess(res);
+
+        traits.ValidateFileList(f, {
+            prefix + "metadata.json",
+            prefix + "SchemaMapping/metadata.json",
+            prefix + "SchemaMapping/mapping.json",
+            prefix + "Table0/metadata.json",
+            prefix + "Table0/scheme.pb",
+            prefix + "Table0/permissions.pb",
+            prefix + "Table0/data_00.csv",
+            prefix + "dir1/dir2/Table2/metadata.json",
+            prefix + "dir1/dir2/Table2/scheme.pb",
+            prefix + "dir1/dir2/Table2/permissions.pb",
+            prefix + "dir1/dir2/Table2/data_00.csv",
+
+            prefix + "metadata.json.sha256",
+            prefix + "SchemaMapping/metadata.json.sha256",
+            prefix + "SchemaMapping/mapping.json.sha256",
+            prefix + "Table0/metadata.json.sha256",
+            prefix + "Table0/scheme.pb.sha256",
+            prefix + "Table0/permissions.pb.sha256",
+            prefix + "Table0/data_00.csv.sha256",
+            prefix + "dir1/dir2/Table2/metadata.json.sha256",
+            prefix + "dir1/dir2/Table2/scheme.pb.sha256",
+            prefix + "dir1/dir2/Table2/permissions.pb.sha256",
+            prefix + "dir1/dir2/Table2/data_00.csv.sha256",
+        });
+    }
+
+    {
+        auto exportSettings = traits.MakeExportSettings(f, "/Root/RecursiveFolderProcessing");
+        traits.SetEncryption(exportSettings, NExport::TEncryptionAlgorithm::AES_128_GCM, "Cool random key!");
+        auto res = traits.Export(f, exportSettings);
+        f.WaitOpStatus(res, EStatus::BAD_REQUEST);
+    }
+
+    {
+        auto importSettings = traits.MakeImportSettings(f, "/Root/RestorePrefix");
+        auto res = traits.Import(f, importSettings);
+        f.WaitOpSuccess(res);
+
+        f.ValidateHasYdbPaths({
+            TBackupTestFixture::TEntryPath::TablePath("/Root/RestorePrefix/Table0", isOlap),
+            TBackupTestFixture::TEntryPath::TablePath("/Root/RestorePrefix/dir1/dir2/Table2", isOlap),
+        });
+    }
+}
+
+template <typename TExportSettings, typename TBackupTestFixture>
 void ExportWithCommonSourcePathAndExplicitTableInsideImpl(TBackupTestFixture& f, bool isOlap) {
     TBackupTraits<TExportSettings> traits;
     const TString prefix = traits.FilePrefix();
@@ -920,6 +983,7 @@ void FilterByPathFailsWhenNoSchemaMappingImpl(TBackupTestFixture& f, bool /*isOl
     const TString prefixRaw = traits.FilePrefixRaw();
     f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableFsBackups(true);
     if constexpr (std::is_same_v<TExportSettings, NExport::TExportToFsSettings>) {
+        f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableExportFiltering(false);
         f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableEncryptedExport(false);
     }
     {
@@ -1096,6 +1160,7 @@ void ExportRecursiveWithoutDestinationPrefixImpl(TBackupTestFixture& f, bool isO
     const TString importPrefix = traits.ImportSrcPrefix();
     f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableFsBackups(true);
     if constexpr (std::is_same_v<TExportSettings, NExport::TExportToFsSettings>) {
+        f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableExportFiltering(false);
         f.Server().GetRuntime()->GetAppData().FeatureFlags.SetEnableEncryptedExport(false);
     }
 
@@ -1761,6 +1826,9 @@ Y_UNIT_TEST_SUITE_F(BackupPathTestFs, TBackupPathTestFixtureFs) {
     Y_UNIT_TEST(ExportWithExcludeRegexps) {
         ExportWithExcludeRegexpsImpl<NExport::TExportToFsSettings, TFsBackupTestFixture>(*this, false);
     }
+    Y_UNIT_TEST(ExportFilteringWithoutEncryption) {
+        ExportFilteringWithoutEncryptionImpl<NExport::TExportToFsSettings, TFsBackupTestFixture>(*this, false);
+    }
     Y_UNIT_TEST(ExportWithCommonSourcePathAndExplicitTableInside) {
         ExportWithCommonSourcePathAndExplicitTableInsideImpl<NExport::TExportToFsSettings, TFsBackupTestFixture>(*this, false);
     }
@@ -1826,6 +1894,10 @@ Y_UNIT_TEST_SUITE_F(BackupPathTest, TBackupPathTestFixture) {
         ExportWithExcludeRegexpsImpl<NExport::TExportToS3Settings, TS3BackupTestFixture>(*this, IsOlap);
     }
 
+    Y_UNIT_TEST_TWIN(ExportFilteringWithoutEncryption, IsOlap) {
+        ExportFilteringWithoutEncryptionImpl<NExport::TExportToS3Settings, TS3BackupTestFixture>(*this, IsOlap);
+    }
+
     Y_UNIT_TEST_TWIN(ImportWithExcludeRegexps, IsOlap) {
         ImportWithExcludeRegexpsImpl<NExport::TExportToS3Settings, TS3BackupTestFixture>(*this, IsOlap);
     }