Add checksums for scheme objects export / import to S3 (#32107)

5 files changed, 45 insertions, 14 deletions

diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
index c05383209cb..a0f2cae847a 100644
--- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
@@ -443,9 +443,7 @@ private:
             const auto databaseRoot = CanonizePath(Self->RootPathElements);
 
             NBackup::TMetadata metadata;
-            // to do: enable view checksum validation
-            constexpr bool EnableChecksums = false;
-            metadata.SetVersion(EnableChecksums ? 1 : 0);
+            metadata.SetVersion(exportInfo.EnableChecksums ? 1 : 0);
             metadata.SetEnablePermissions(exportInfo.EnablePermissions);
 
             TMaybe<NBackup::TEncryptionIV> iv;
@@ -455,7 +453,8 @@ private:
 
             item.SchemeUploader = ctx.Register(CreateSchemeUploader(
                 Self->SelfId(), exportInfo.Id, itemIdx, item.SourcePathId,
-                exportSettings, databaseRoot, metadata.Serialize(), exportInfo.EnablePermissions,
+                exportSettings, databaseRoot, metadata.Serialize(),
+                exportInfo.EnablePermissions, exportInfo.EnableChecksums,
                 iv
             ));
             Self->RunningExportSchemeUploaders.emplace(item.SchemeUploader);
diff --git a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp
index 6f9c0299d3c..3ae06cc99f8 100644
--- a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp
@@ -244,6 +244,12 @@ class TSchemeUploader: public TExportFilesUploader<TSchemeUploader> {
             return;
         }
 
+        if (EnableChecksums) {
+            if (!AddFile(NBackup::ChecksumKey(FileName), NBackup::ComputeChecksum(Scheme))) {
+                return;
+            }
+        }
+
         if (EnablePermissions) {
             if (!Permissions) {
                 return Finish(false, "cannot infer permissions");
@@ -252,6 +258,12 @@ class TSchemeUploader: public TExportFilesUploader<TSchemeUploader> {
             if (!AddFile("permissions.pb", Permissions, MakeIV(NBackup::EBackupFileType::Permissions))) {
                 return;
             }
+
+            if (EnableChecksums) {
+                if (!AddFile(NBackup::ChecksumKey("permissions.pb"), NBackup::ComputeChecksum(Permissions))) {
+                    return;
+                }
+            }
         }
 
         if (!Metadata) {
@@ -262,6 +274,12 @@ class TSchemeUploader: public TExportFilesUploader<TSchemeUploader> {
             return;
         }
 
+        if (EnableChecksums) {
+            if (!AddFile(NBackup::ChecksumKey("metadata.json"), NBackup::ComputeChecksum(Metadata))) {
+                return;
+            }
+        }
+
         UploadFiles();
     }
 
@@ -305,6 +323,7 @@ public:
         const TString& databaseRoot,
         const TString& metadata,
         bool enablePermissions,
+        bool enableChecksums,
         const TMaybe<NBackup::TEncryptionIV>& iv
     )
         : TExportFilesUploader<TSchemeUploader>(settings, GetDestinationPrefix(settings, itemIdx))
@@ -315,6 +334,7 @@ public:
         , IV(iv)
         , DatabaseRoot(databaseRoot)
         , EnablePermissions(enablePermissions)
+        , EnableChecksums(enableChecksums)
         , Metadata(metadata)
     {
     }
@@ -344,7 +364,8 @@ private:
 
     TString DatabaseRoot;
     TString Scheme;
-    bool EnablePermissions = false;
+    const bool EnablePermissions;
+    const bool EnableChecksums;
     TString Permissions;
     TString Metadata;
 }; // TSchemeUploader
@@ -463,10 +484,10 @@ private:
 
 IActor* CreateSchemeUploader(TActorId schemeShard, ui64 exportId, ui32 itemIdx, TPathId sourcePathId,
     const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot, const TString& metadata,
-    bool enablePermissions, const TMaybe<NBackup::TEncryptionIV>& iv
+    bool enablePermissions, bool enableChecksums, const TMaybe<NBackup::TEncryptionIV>& iv
 ) {
     return new TSchemeUploader(schemeShard, exportId, itemIdx, sourcePathId, settings, databaseRoot,
-        metadata, enablePermissions, iv);
+        metadata, enablePermissions, enableChecksums, iv);
 }
 
 NActors::IActor* CreateExportMetadataUploader(NActors::TActorId schemeShard, ui64 exportId,
diff --git a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.h b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.h
index 7a03bfd9899..ae64d57c2ec 100644
--- a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.h
+++ b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.h
@@ -20,7 +20,7 @@ namespace NKikimr::NSchemeShard {
 
 NActors::IActor* CreateSchemeUploader(NActors::TActorId schemeShard, ui64 exportId, ui32 itemIdx, TPathId sourcePathId,
     const Ydb::Export::ExportToS3Settings& settings, const TString& databaseRoot, const TString& metadata,
-    bool enablePermissions, const TMaybe<NBackup::TEncryptionIV>& iv
+    bool enablePermissions, bool enableChecksums, const TMaybe<NBackup::TEncryptionIV>& iv
 );
 
 NActors::IActor* CreateExportMetadataUploader(NActors::TActorId schemeShard, ui64 exportId,
diff --git a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp
index 7c0b2845218..97e68e7d620 100644
--- a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp
+++ b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp
@@ -694,6 +694,11 @@ namespace {
             }
         }
 
+        void CheckPathWithChecksum(const TString& path) {
+            UNIT_ASSERT(HasS3File(path));
+            UNIT_ASSERT(HasS3File(path + ".sha256"));
+        }
+
         void TestReplication(const TString& scheme, const TString& expected) {
             auto options = TTestEnvOptions()
                 .InitYdbDriver(true);
@@ -719,13 +724,13 @@ namespace {
 
             TestGetExport(Runtime(), txId, "/MyRoot", Ydb::StatusIds::SUCCESS);
 
-            UNIT_ASSERT(HasS3File("/Replication/create_async_replication.sql"));
+            CheckPathWithChecksum("/Replication/create_async_replication.sql");
             const auto content = GetS3FileContent("/Replication/create_async_replication.sql");
             UNIT_ASSERT_EQUAL_C(
                 content, expected,
                 TStringBuilder() << "\nExpected:\n\n" << expected << "\n\nActual:\n\n" << content);
 
-            UNIT_ASSERT(HasS3File("/Replication/permissions.pb"));
+            CheckPathWithChecksum("/Replication/permissions.pb");
             const auto permissions = GetS3FileContent("/Replication/permissions.pb");
             const auto permissions_expected = "actions {\n  change_owner: \"root@builtin\"\n}\n";
             UNIT_ASSERT_EQUAL_C(
@@ -771,13 +776,13 @@ namespace {
 
             TestGetExport(Runtime(), txId, "/MyRoot", Ydb::StatusIds::SUCCESS);
 
-            UNIT_ASSERT(HasS3File("/Transfer/create_transfer.sql"));
+            CheckPathWithChecksum("/Transfer/create_transfer.sql");
             const auto content = GetS3FileContent("/Transfer/create_transfer.sql");
             UNIT_ASSERT_EQUAL_C(
                 content, expected,
                 TStringBuilder() << "\nExpected:\n\n" << expected << "\n\nActual:\n\n" << content);
 
-            UNIT_ASSERT(HasS3File("/Transfer/permissions.pb"));
+            CheckPathWithChecksum("/Transfer/permissions.pb");
             const auto permissions = GetS3FileContent("/Transfer/permissions.pb");
             const auto permissions_expected = "actions {\n  change_owner: \"root@builtin\"\n}\n";
             UNIT_ASSERT_EQUAL_C(
@@ -812,7 +817,7 @@ namespace {
 
             TestGetExport(Runtime(), txId, "/MyRoot", Ydb::StatusIds::SUCCESS);
 
-            UNIT_ASSERT(HasS3File("/DataSource/create_external_data_source.sql"));
+            CheckPathWithChecksum("/DataSource/create_external_data_source.sql");
             const auto content = GetS3FileContent("/DataSource/create_external_data_source.sql");
 
             TString expectedHeader = "-- database: \"/MyRoot\"\n"
@@ -835,7 +840,7 @@ namespace {
                 static_cast<long>(expectedProperties.size()) - 1,
                 "Properties count mismatch");
 
-            UNIT_ASSERT(HasS3File("/DataSource/permissions.pb"));
+            CheckPathWithChecksum("/DataSource/permissions.pb");
             const auto permissions = GetS3FileContent("/DataSource/permissions.pb");
             const auto permissions_expected = "actions {\n  change_owner: \"root@builtin\"\n}\n";
             UNIT_ASSERT_EQUAL_C(
diff --git a/ydb/services/ydb/backup_ut/encrypted_backup_ut.cpp b/ydb/services/ydb/backup_ut/encrypted_backup_ut.cpp
index 34fff739273..6930dc1c2bb 100644
--- a/ydb/services/ydb/backup_ut/encrypted_backup_ut.cpp
+++ b/ydb/services/ydb/backup_ut/encrypted_backup_ut.cpp
@@ -685,8 +685,11 @@ Y_UNIT_TEST_SUITE_F(EncryptedExportTest, TBackupEncryptionTestFixture) {
                 "/test_bucket/Prefix/SchemaMapping/mapping.json.enc",
                 "/test_bucket/Prefix/SchemaMapping/mapping.json.sha256",
                 "/test_bucket/Prefix/001/create_topic.pb.enc",
+                "/test_bucket/Prefix/001/create_topic.pb.sha256",
                 "/test_bucket/Prefix/001/metadata.json.enc",
+                "/test_bucket/Prefix/001/metadata.json.sha256",
                 "/test_bucket/Prefix/001/permissions.pb.enc",
+                "/test_bucket/Prefix/001/permissions.pb.sha256",
             });
         }
 
@@ -732,8 +735,11 @@ Y_UNIT_TEST_SUITE_F(EncryptedExportTest, TBackupEncryptionTestFixture) {
                 "/test_bucket/Prefix/SchemaMapping/mapping.json.enc",
                 "/test_bucket/Prefix/SchemaMapping/mapping.json.sha256",
                 "/test_bucket/Prefix/001/create_view.sql.enc",
+                "/test_bucket/Prefix/001/create_view.sql.sha256",
                 "/test_bucket/Prefix/001/metadata.json.enc",
+                "/test_bucket/Prefix/001/metadata.json.sha256",
                 "/test_bucket/Prefix/001/permissions.pb.enc",
+                "/test_bucket/Prefix/001/permissions.pb.sha256",
             });
         }