aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIlia Shakhov <pixcc@ydb.tech>2025-02-10 21:16:55 +0300
committerGitHub <noreply@github.com>2025-02-10 21:16:55 +0300
commit80d32f2f8ea4092e4cff97e64f6a149621a8c956 (patch)
tree400e9ee83a4853c3d069c52afa9b4ab6bcdf369f
parent4878da3dd9b3b683b5420c7dbb403fe035ff74cd (diff)
downloadydb-80d32f2f8ea4092e4cff97e64f6a149621a8c956.tar.gz
Add cluster & database dump to YDB CLI (#14306)
-rw-r--r--ydb/library/backup/backup.cpp353
-rw-r--r--ydb/library/backup/backup.h3
-rw-r--r--ydb/library/backup/ya.make1
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_admin.cpp35
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_admin.h11
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp39
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_cluster.h11
-rw-r--r--ydb/public/lib/ydb_cli/common/command.cpp2
-rw-r--r--ydb/public/lib/ydb_cli/common/command.h18
-rw-r--r--ydb/public/lib/ydb_cli/dump/dump.cpp18
-rw-r--r--ydb/public/lib/ydb_cli/dump/dump.h4
-rw-r--r--ydb/public/lib/ydb_cli/dump/dump_impl.cpp24
-rw-r--r--ydb/public/lib/ydb_cli/dump/dump_impl.h4
-rw-r--r--ydb/public/lib/ydb_cli/dump/files/files.cpp24
-rw-r--r--ydb/public/lib/ydb_cli/dump/files/files.h4
-rw-r--r--ydb/public/lib/ydb_cli/dump/util/util.cpp19
-rw-r--r--ydb/public/lib/ydb_cli/dump/util/util.h20
-rw-r--r--ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h13
-rw-r--r--ydb/public/sdk/cpp/src/client/cms/cms.cpp76
-rw-r--r--ydb/tests/functional/ydb_cli/test_ydb_backup.py160
20 files changed, 803 insertions, 36 deletions
diff --git a/ydb/library/backup/backup.cpp b/ydb/library/backup/backup.cpp
index 6c014a2874..7024072ac2 100644
--- a/ydb/library/backup/backup.cpp
+++ b/ydb/library/backup/backup.cpp
@@ -2,7 +2,16 @@
#include "db_iterator.h"
#include "util.h"
+#include <ydb-cpp-sdk/client/cms/cms.h>
+#include <ydb-cpp-sdk/client/draft/ydb_view.h>
+#include <ydb-cpp-sdk/client/driver/driver.h>
+#include <ydb-cpp-sdk/client/proto/accessor.h>
+#include <ydb-cpp-sdk/client/result/result.h>
+#include <ydb-cpp-sdk/client/table/table.h>
+#include <ydb-cpp-sdk/client/topic/client.h>
+#include <ydb-cpp-sdk/client/value/value.h>
#include <ydb/public/api/protos/draft/ydb_view.pb.h>
+#include <ydb/public/api/protos/ydb_cms.pb.h>
#include <ydb/public/api/protos/ydb_rate_limiter.pb.h>
#include <ydb/public/api/protos/ydb_table.pb.h>
#include <ydb/public/lib/ydb_cli/common/recursive_remove.h>
@@ -676,9 +685,9 @@ void BackupCoordinationNode(TDriver driver, const TString& dbPath, const TFsPath
void CreateClusterDirectory(const TDriver& driver, const TString& path, bool rootBackupDir = false) {
if (rootBackupDir) {
- LOG_I("Create temporary directory " << path.Quote());
+ LOG_I("Create temporary directory " << path.Quote() << " in database");
} else {
- LOG_D("Create directory " << path.Quote());
+ LOG_D("Create directory " << path.Quote() << " in database");
}
NScheme::TSchemeClient client(driver);
TStatus status = client.MakeDirectory(path).GetValueSync();
@@ -693,7 +702,7 @@ void RemoveClusterDirectory(const TDriver& driver, const TString& path) {
}
void RemoveClusterDirectoryRecursive(const TDriver& driver, const TString& path) {
- LOG_I("Remove temporary directory " << path.Quote());
+ LOG_I("Remove temporary directory " << path.Quote() << " in database");
NScheme::TSchemeClient schemeClient(driver);
NTable::TTableClient tableClient(driver);
TStatus status = NConsoleClient::RemoveDirectoryRecursive(schemeClient, tableClient, path, {}, true, false);
@@ -842,6 +851,281 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
}
+namespace {
+
+NCms::TListDatabasesResult ListDatabases(TDriver driver) {
+ NCms::TCmsClient client(driver);
+
+ auto status = NDump::ListDatabases(client);
+ VerifyStatus(status);
+
+ return status;
+}
+
+NCms::TGetDatabaseStatusResult GetDatabaseStatus(TDriver driver, const std::string& path) {
+ NCms::TCmsClient client(driver);
+
+ auto status = NDump::GetDatabaseStatus(client, path);
+ VerifyStatus(status);
+
+ return status;
+}
+
+bool IsNotLowerAlphaNum(char c) {
+ if (isalnum(c)) {
+ if (isalpha(c)) {
+ return !islower(c);
+ }
+
+ return false;
+ }
+
+ return true;
+}
+
+bool IsValidSid(const std::string& sid) {
+ return std::find_if(sid.begin(), sid.end(), IsNotLowerAlphaNum) == sid.end();
+}
+
+struct TAdmins {
+ TString GroupSid;
+ THashSet<TString> UserSids;
+};
+
+TAdmins FindAdmins(TDriver driver, const TString& dbPath) {
+ THashSet<TString> adminUserSids;
+
+ auto entry = DescribePath(driver, dbPath);
+
+ NYdb::NTable::TTableClient client(driver);
+ auto query = Sprintf("SELECT * FROM `%s/.sys/auth_group_members`", dbPath.c_str());
+ auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx();
+
+ std::vector<TResultSet> resultSets;
+ TStatus status = client.RetryOperationSync([&](NTable::TSession session) {
+ auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync();
+ VerifyStatus(result);
+ resultSets = result.GetResultSets();
+ return result;
+ });
+ VerifyStatus(status);
+
+ TStringStream alterGroupQuery;
+ for (const auto& resultSet : resultSets) {
+ TResultSetParser parser(resultSet);
+ while (parser.TryNextRow()) {
+ auto groupSidValue = parser.GetValue("GroupSid");
+ auto memberSidValue = parser.GetValue("MemberSid");
+
+ const auto& groupSid = groupSidValue.GetProto().text_value();
+ const auto& memberSid = memberSidValue.GetProto().text_value();
+
+ if (groupSid == entry.Owner) {
+ adminUserSids.insert(memberSid);
+ }
+ }
+ }
+
+ return { TString(entry.Owner), adminUserSids };
+}
+
+struct TBackupDatabaseSettings {
+ bool WithRegularUsers = false;
+ bool WithContent = false;
+ TString TemporalBackupPostfix;
+};
+
+void BackupUsers(TDriver driver, const TString& dbPath, const TFsPath& folderPath, const THashSet<TString>& filter = {}) {
+ NYdb::NTable::TTableClient client(driver);
+ auto query = Sprintf("SELECT * FROM `%s/.sys/auth_users`", dbPath.c_str());
+ auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx();
+
+ std::vector<TResultSet> resultSets;
+ TStatus status = client.RetryOperationSync([&](NTable::TSession session) {
+ auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync();
+ VerifyStatus(result);
+ resultSets = result.GetResultSets();
+ return result;
+ });
+ VerifyStatus(status);
+
+ TStringStream createUserQuery;
+ for (const auto& resultSet : resultSets) {
+ TResultSetParser parser(resultSet);
+ while (parser.TryNextRow()) {
+ auto sidValue = parser.GetValue("Sid");
+ auto passwordValue = parser.GetValue("PasswordHash");
+ auto isEnabledValue = parser.GetValue("IsEnabled");
+
+ const auto& sid = sidValue.GetProto().text_value();
+ const auto& password = passwordValue.GetProto().text_value();
+ bool isEnabled = isEnabledValue.GetProto().bool_value();
+
+ if (!filter.empty() && !filter.contains(sid)) {
+ continue;
+ }
+
+ // Some SIDs may be created through configuration that bypasses this checks
+ if (IsValidSid(sid)) {
+ createUserQuery << Sprintf("CREATE USER `%s` HASH '%s';\n", sid.c_str(), password.c_str());
+ }
+
+ if (!isEnabled) {
+ createUserQuery << Sprintf("ALTER USER `%s` NOLOGIN;\n", sid.c_str());
+ }
+ }
+ }
+
+ WriteCreationQueryToFile(createUserQuery.Str(), folderPath, NDump::NFiles::CreateUser());
+}
+
+void BackupGroups(TDriver driver, const TString& dbPath, const TFsPath& folderPath, const THashSet<TString>& filter = {}) {
+ NYdb::NTable::TTableClient client(driver);
+ auto query = Sprintf("SELECT * FROM `%s/.sys/auth_groups`", dbPath.c_str());
+ auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx();
+
+ std::vector<TResultSet> resultSets;
+ TStatus status = client.RetryOperationSync([&](NTable::TSession session) {
+ auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync();
+ VerifyStatus(result);
+ resultSets = result.GetResultSets();
+ return result;
+ });
+ VerifyStatus(status);
+
+ TStringStream createGroupQuery;
+ for (const auto& resultSet : resultSets) {
+ TResultSetParser parser(resultSet);
+ while (parser.TryNextRow()) {
+ auto sidValue = parser.GetValue("Sid");
+ const auto& sid = sidValue.GetProto().text_value();
+
+ if (!filter.empty() && !filter.contains(sid)) {
+ continue;
+ }
+
+ // Some SIDs may be created through configuration that bypasses this checks
+ if (IsValidSid(sid)) {
+ createGroupQuery << Sprintf("CREATE GROUP `%s`;\n", sid.c_str());
+ }
+ }
+ }
+
+ WriteCreationQueryToFile(createGroupQuery.Str(), folderPath, NDump::NFiles::CreateGroup());
+}
+
+void BackupGroupMembers(TDriver driver, const TString& dbPath, const TFsPath& folderPath, const THashSet<TString>& filterGroups = {}) {
+ NYdb::NTable::TTableClient client(driver);
+ auto query = Sprintf("SELECT * FROM `%s/.sys/auth_group_members`", dbPath.c_str());
+ auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx();
+
+ std::vector<TResultSet> resultSets;
+ TStatus status = client.RetryOperationSync([&](NTable::TSession session) {
+ auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync();
+ VerifyStatus(result);
+ resultSets = result.GetResultSets();
+ return result;
+ });
+ VerifyStatus(status);
+
+ TStringStream alterGroupQuery;
+ for (const auto& resultSet : resultSets) {
+ TResultSetParser parser(resultSet);
+ while (parser.TryNextRow()) {
+ auto groupSidValue = parser.GetValue("GroupSid");
+ auto memberSidValue = parser.GetValue("MemberSid");
+
+ const auto& groupSid = groupSidValue.GetProto().text_value();
+ const auto& memberSid = memberSidValue.GetProto().text_value();
+
+ if (!filterGroups.empty() && !filterGroups.contains(groupSid)) {
+ continue;
+ }
+
+ alterGroupQuery << Sprintf("ALTER GROUP `%s` ADD USER `%s`;\n", groupSid.c_str(), memberSid.c_str());
+ }
+ }
+
+ WriteCreationQueryToFile(alterGroupQuery.Str(), folderPath, NDump::NFiles::AlterGroup());
+}
+
+void BackupDatabaseImpl(TDriver driver, const TString& dbPath, const TFsPath& folderPath, TBackupDatabaseSettings settings) {
+ LOG_I("Backup database " << dbPath.Quote() << " to " << folderPath.GetPath().Quote());
+ folderPath.MkDirs();
+
+ auto status = GetDatabaseStatus(driver, dbPath);
+ Ydb::Cms::CreateDatabaseRequest proto;
+ status.SerializeTo(proto);
+ WriteProtoToFile(proto, folderPath, NDump::NFiles::Database());
+
+ if (!settings.WithRegularUsers) {
+ TAdmins admins = FindAdmins(driver, dbPath);
+ BackupUsers(driver, dbPath, folderPath, admins.UserSids);
+ BackupGroups(driver, dbPath, folderPath, { admins.GroupSid });
+ BackupGroupMembers(driver, dbPath, folderPath, { admins.GroupSid });
+ } else {
+ BackupUsers(driver, dbPath, folderPath);
+ BackupGroups(driver, dbPath, folderPath);
+ BackupGroupMembers(driver, dbPath, folderPath);
+ }
+
+ BackupPermissions(driver, dbPath, "", folderPath);
+ if (settings.WithContent) {
+ // full path to temporal directory in database
+ TString tmpDbFolder;
+ try {
+ tmpDbFolder = JoinDatabasePath(dbPath, "~" + settings.TemporalBackupPostfix);
+ CreateClusterDirectory(driver, tmpDbFolder, true);
+
+ NYql::TIssues issues;
+ BackupFolderImpl(
+ driver,
+ dbPath,
+ tmpDbFolder,
+ folderPath,
+ /* exclusionPatterns */ {},
+ /* schemaOnly */ false,
+ /* useConsistentCopyTable */ true,
+ /* avoidCopy */ false,
+ /* preservePoolKinds */ false,
+ /* ordered */ false,
+ issues
+ );
+
+ if (issues) {
+ Cerr << issues.ToString();
+ }
+ } catch (...) {
+ RemoveClusterDirectoryRecursive(driver, tmpDbFolder);
+ folderPath.ForceDelete();
+ throw;
+ }
+ RemoveClusterDirectoryRecursive(driver, tmpDbFolder);
+ }
+}
+
+TString FindClusterRootPath(TDriver driver) {
+ NScheme::TSchemeClient client(driver);
+ auto status = NDump::ListDirectory(client, "/");
+ VerifyStatus(status);
+
+ Y_ENSURE(status.GetChildren().size() == 1, "Exactly one cluster root expected, found: " << JoinSeq(", ", status.GetChildren()));
+ return "/" + status.GetChildren().begin()->Name;
+}
+
+void BackupClusterRoot(TDriver driver, const TFsPath& folderPath) {
+ TString rootPath = FindClusterRootPath(driver);
+
+ LOG_I("Backup cluster root " << rootPath.Quote() << " to " << folderPath);
+
+ BackupUsers(driver, rootPath, folderPath);
+ BackupGroups(driver, rootPath, folderPath);
+ BackupGroupMembers(driver, rootPath, folderPath);
+ BackupPermissions(driver, rootPath, "", folderPath);
+}
+
+} // anonymous namespace
+
void CheckedCreateBackupFolder(const TFsPath& folderPath) {
const bool exists = folderPath.Exists();
if (exists) {
@@ -906,4 +1190,67 @@ void BackupFolder(const TDriver& driver, const TString& database, const TString&
LOG_I("Backup completed successfully");
}
+void BackupDatabase(const TDriver& driver, const TString& database, TFsPath folderPath) {
+ TString temporalBackupPostfix = CreateTemporalBackupName();
+ if (!folderPath) {
+ folderPath = temporalBackupPostfix;
+ }
+ CheckedCreateBackupFolder(folderPath);
+
+ try {
+ NYql::TIssues issues;
+ TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways);
+
+ BackupDatabaseImpl(driver, database, folderPath, {
+ .WithRegularUsers = true,
+ .WithContent = true,
+ .TemporalBackupPostfix = temporalBackupPostfix,
+ });
+
+ folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
+ if (issues) {
+ Cerr << issues.ToString();
+ }
+ } catch (...) {
+ LOG_E("Backup failed");
+ folderPath.ForceDelete();
+ throw;
+ }
+ LOG_I("Backup database " << database.Quote() << " is completed successfully");
+}
+
+void BackupCluster(const TDriver& driver, TFsPath folderPath) {
+ TString temporalBackupPostfix = CreateTemporalBackupName();
+ if (!folderPath) {
+ folderPath = temporalBackupPostfix;
+ }
+ CheckedCreateBackupFolder(folderPath);
+
+ LOG_I("Backup cluster to " << folderPath.GetPath().Quote());
+
+ try {
+ NYql::TIssues issues;
+ TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways);
+
+ BackupClusterRoot(driver, folderPath);
+ auto databases = ListDatabases(driver);
+ for (const auto& database : databases.GetPaths()) {
+ BackupDatabaseImpl(driver, TString(database), folderPath.Child("." + database), {
+ .WithRegularUsers = false,
+ .WithContent = false,
+ });
+ }
+
+ folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists();
+ if (issues) {
+ Cerr << issues.ToString();
+ }
+ } catch (...) {
+ LOG_E("Backup failed");
+ folderPath.ForceDelete();
+ throw;
+ }
+ LOG_I("Backup cluster is completed successfully");
+}
+
} // NYdb::NBackup
diff --git a/ydb/library/backup/backup.h b/ydb/library/backup/backup.h
index dfd96b40b3..c93c20d884 100644
--- a/ydb/library/backup/backup.h
+++ b/ydb/library/backup/backup.h
@@ -46,6 +46,9 @@ void BackupFolder(
bool preservePoolKinds = false,
bool ordered = false);
+void BackupCluster(const TDriver& driver, TFsPath folderPath);
+void BackupDatabase(const TDriver& driver, const TString& database, TFsPath folderPath);
+
// For unit-tests only
TMaybe<TValue> ProcessResultSet(
TStringStream& ss,
diff --git a/ydb/library/backup/ya.make b/ydb/library/backup/ya.make
index ee77236af7..0507feabcc 100644
--- a/ydb/library/backup/ya.make
+++ b/ydb/library/backup/ya.make
@@ -12,6 +12,7 @@ PEERDIR(
ydb/public/lib/ydb_cli/dump/util
ydb/public/lib/yson_value
ydb/public/lib/ydb_cli/dump/files
+ ydb/public/sdk/cpp/src/client/cms
ydb/public/sdk/cpp/src/client/coordination
ydb/public/sdk/cpp/src/client/draft
ydb/public/sdk/cpp/src/client/driver
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp
index 6c7beb1f13..21208491ef 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp
@@ -5,6 +5,11 @@
#include "ydb_cluster.h"
#include <ydb/public/lib/ydb_cli/common/command_utils.h>
+#include <ydb/public/lib/ydb_cli/dump/dump.h>
+
+#define INCLUDE_YDB_INTERNAL_H
+#include <ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h>
+#undef INCLUDE_YDB_INTERNAL_H
namespace NYdb {
namespace NConsoleClient {
@@ -24,9 +29,39 @@ public:
: TClientCommandTree("database", {}, "Database-wide administration")
{
AddCommand(std::make_unique<NDynamicConfig::TCommandConfig>());
+ AddCommand(std::make_unique<TCommandDatabaseDump>());
}
};
+TCommandDatabaseDump::TCommandDatabaseDump()
+ : TYdbCommand("dump", {}, "Dump database into local directory")
+{}
+
+void TCommandDatabaseDump::Config(TConfig& config) {
+ TYdbCommand::Config(config);
+ config.SetFreeArgsNum(0);
+
+ config.Opts->AddLongOption('o', "output", "Path in a local filesystem to a directory to place dump into."
+ " Directory should either not exist or be empty."
+ " If not specified, the dump is placed in the directory backup_YYYYYYMMDDDThhmmss.")
+ .RequiredArgument("PATH")
+ .StoreResult(&FilePath);
+}
+
+void TCommandDatabaseDump::Parse(TConfig& config) {
+ TClientCommand::Parse(config);
+}
+
+int TCommandDatabaseDump::Run(TConfig& config) {
+ auto log = std::make_shared<TLog>(CreateLogBackend("cerr", TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)));
+ log->SetFormatter(GetPrefixLogFormatter(""));
+
+ NDump::TClient client(CreateDriver(config), std::move(log));
+ NStatusHelpers::ThrowOnErrorOrPrintIssues(client.DumpDatabase(config.Database, FilePath));
+
+ return EXIT_SUCCESS;
+}
+
TCommandAdmin::TCommandAdmin()
: TClientCommandTree("admin", {}, "Administrative cluster operations")
{
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.h b/ydb/public/lib/ydb_cli/commands/ydb_admin.h
index d544544ec9..8e0d65d836 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_admin.h
+++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.h
@@ -12,5 +12,16 @@ protected:
virtual void Config(TConfig& config) override;
};
+class TCommandDatabaseDump : public TYdbCommand {
+public:
+ TCommandDatabaseDump();
+ void Config(TConfig& config) override;
+ void Parse(TConfig& config) override;
+ int Run(TConfig& config) override;
+
+private:
+ TString FilePath;
+};
+
}
}
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp
index 50c3c3e66b..959873ab88 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp
@@ -1,8 +1,14 @@
#include "ydb_cluster.h"
-#include <ydb-cpp-sdk/client/bsconfig/storage_config.h>
#include "ydb_dynamic_config.h"
+#include <ydb-cpp-sdk/client/bsconfig/storage_config.h>
+#include <ydb/public/lib/ydb_cli/dump/dump.h>
+
+#define INCLUDE_YDB_INTERNAL_H
+#include <ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h>
+#undef INCLUDE_YDB_INTERNAL_H
+
using namespace NKikimr;
namespace NYdb::NConsoleClient::NCluster {
@@ -12,6 +18,7 @@ TCommandCluster::TCommandCluster()
{
AddCommand(std::make_unique<TCommandClusterBootstrap>());
AddCommand(std::make_unique<NDynamicConfig::TCommandConfig>(true));
+ AddCommand(std::make_unique<TCommandClusterDump>());
}
TCommandClusterBootstrap::TCommandClusterBootstrap()
@@ -37,4 +44,34 @@ int TCommandClusterBootstrap::Run(TConfig& config) {
return EXIT_SUCCESS;
}
+TCommandClusterDump::TCommandClusterDump()
+ : TYdbCommand("dump", {}, "Dump cluster into local directory")
+{}
+
+void TCommandClusterDump::Config(TConfig& config) {
+ TYdbCommand::Config(config);
+ config.SetFreeArgsNum(0);
+ config.AllowEmptyDatabase = true;
+
+ config.Opts->AddLongOption('o', "output", "Path in a local filesystem to a directory to place dump into."
+ " Directory should either not exist or be empty."
+ " If not specified, the dump is placed in the directory backup_YYYYYYMMDDDThhmmss.")
+ .RequiredArgument("PATH")
+ .StoreResult(&FilePath);
+}
+
+void TCommandClusterDump::Parse(TConfig& config) {
+ TClientCommand::Parse(config);
+}
+
+int TCommandClusterDump::Run(TConfig& config) {
+ auto log = std::make_shared<TLog>(CreateLogBackend("cerr", TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel)));
+ log->SetFormatter(GetPrefixLogFormatter(""));
+
+ NDump::TClient client(CreateDriver(config), std::move(log));
+ NStatusHelpers::ThrowOnErrorOrPrintIssues(client.DumpCluster(FilePath));
+
+ return EXIT_SUCCESS;
+}
+
} // namespace NYdb::NConsoleClient::NCluster
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.h b/ydb/public/lib/ydb_cli/commands/ydb_cluster.h
index 6a5339979c..ebe4375fde 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.h
+++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.h
@@ -22,4 +22,15 @@ public:
int Run(TConfig& config) override;
};
+class TCommandClusterDump : public TYdbCommand {
+public:
+ TCommandClusterDump();
+ void Config(TConfig& config) override;
+ void Parse(TConfig& config) override;
+ int Run(TConfig& config) override;
+
+private:
+ TString FilePath;
+};
+
} // namespace NYdb::NConsoleClient::NCluster
diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp
index 5a1b71c537..a4960f999a 100644
--- a/ydb/public/lib/ydb_cli/common/command.cpp
+++ b/ydb/public/lib/ydb_cli/common/command.cpp
@@ -161,7 +161,7 @@ void TClientCommand::SaveParseResult(TConfig& config) {
}
void TClientCommand::Prepare(TConfig& config) {
- config.ArgsSettings.Reset(new TConfig::TArgSettings());
+ config.ArgsSettings = TConfig::TArgSettings();
config.Opts = &Opts;
Config(config);
CheckForExecutableOptions(config);
diff --git a/ydb/public/lib/ydb_cli/common/command.h b/ydb/public/lib/ydb_cli/common/command.h
index 412ab9b9b8..abf6ad6bd4 100644
--- a/ydb/public/lib/ydb_cli/common/command.h
+++ b/ydb/public/lib/ydb_cli/common/command.h
@@ -104,7 +104,7 @@ public:
THashSet<TString> ExecutableOptions;
bool HasExecutableOptions = false;
TString Path;
- THolder<TArgSettings> ArgsSettings;
+ TArgSettings ArgsSettings;
TString Address;
TString Database;
TString CaCerts;
@@ -185,18 +185,18 @@ public:
}
void SetFreeArgsMin(size_t value) {
- ArgsSettings->Min.Set(value);
+ ArgsSettings.Min.Set(value);
Opts->SetFreeArgsMin(value);
}
void SetFreeArgsMax(size_t value) {
- ArgsSettings->Max.Set(value);
+ ArgsSettings.Max.Set(value);
Opts->SetFreeArgsMax(value);
}
void SetFreeArgsNum(size_t minValue, size_t maxValue) {
- ArgsSettings->Min.Set(minValue);
- ArgsSettings->Max.Set(maxValue);
+ ArgsSettings.Min.Set(minValue);
+ ArgsSettings.Max.Set(maxValue);
Opts->SetFreeArgsNum(minValue, maxValue);
}
@@ -209,10 +209,10 @@ public:
if (HasHelpCommand() || HasExecutableOptions) {
return;
}
- bool minSet = ArgsSettings->Min.GetIsSet();
- size_t minValue = ArgsSettings->Min.Get();
- bool maxSet = ArgsSettings->Max.GetIsSet();
- size_t maxValue = ArgsSettings->Max.Get();
+ bool minSet = ArgsSettings.Min.GetIsSet();
+ size_t minValue = ArgsSettings.Min.Get();
+ bool maxSet = ArgsSettings.Max.GetIsSet();
+ size_t maxValue = ArgsSettings.Max.Get();
bool minFailed = minSet && count < minValue;
bool maxFailed = maxSet && count > maxValue;
if (minFailed || maxFailed) {
diff --git a/ydb/public/lib/ydb_cli/dump/dump.cpp b/ydb/public/lib/ydb_cli/dump/dump.cpp
index c4acb28619..cbaa7b41b4 100644
--- a/ydb/public/lib/ydb_cli/dump/dump.cpp
+++ b/ydb/public/lib/ydb_cli/dump/dump.cpp
@@ -33,6 +33,16 @@ public:
return client.Restore(fsPath, dbPath, settings);
}
+ TDumpResult DumpCluster(const TString& fsPath) {
+ auto client = TDumpClient(Driver, Log);
+ return client.DumpCluster(fsPath);
+ }
+
+ TDumpResult DumpDatabase(const TString& database, const TString& fsPath) {
+ auto client = TDumpClient(Driver, Log);
+ return client.DumpDatabase(database, fsPath);
+ }
+
private:
const TDriver Driver;
std::shared_ptr<TLog> Log;
@@ -67,5 +77,13 @@ TRestoreResult TClient::Restore(const TString& fsPath, const TString& dbPath, co
return Impl_->Restore(fsPath, dbPath, settings);
}
+TDumpResult TClient::DumpCluster(const TString& fsPath) {
+ return Impl_->DumpCluster(fsPath);
+}
+
+TDumpResult TClient::DumpDatabase(const TString& database, const TString& fsPath) {
+ return Impl_->DumpDatabase(database, fsPath);
+}
+
} // NDump
} // NYdb
diff --git a/ydb/public/lib/ydb_cli/dump/dump.h b/ydb/public/lib/ydb_cli/dump/dump.h
index 1f85a88230..1c5374c3f0 100644
--- a/ydb/public/lib/ydb_cli/dump/dump.h
+++ b/ydb/public/lib/ydb_cli/dump/dump.h
@@ -130,6 +130,10 @@ public:
TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {});
TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {});
+ TDumpResult DumpCluster(const TString& fsPath);
+
+ TDumpResult DumpDatabase(const TString& database, const TString& fsPath);
+
private:
std::shared_ptr<TImpl> Impl_;
diff --git a/ydb/public/lib/ydb_cli/dump/dump_impl.cpp b/ydb/public/lib/ydb_cli/dump/dump_impl.cpp
index b92342d231..be3eb896eb 100644
--- a/ydb/public/lib/ydb_cli/dump/dump_impl.cpp
+++ b/ydb/public/lib/ydb_cli/dump/dump_impl.cpp
@@ -36,5 +36,29 @@ TDumpResult TDumpClient::Dump(const TString& dbPath, const TString& fsPath, cons
}
}
+TDumpResult TDumpClient::DumpCluster(const TString& fsPath) {
+ try {
+ NBackup::SetLog(Log);
+ NBackup::BackupCluster(Driver, fsPath);
+ return Result<TDumpResult>();
+ } catch (NBackup::TYdbErrorException& e) {
+ return TDumpResult(std::move(e.Status));
+ } catch (const yexception& e) {
+ return Result<TDumpResult>(EStatus::INTERNAL_ERROR, e.what());
+ }
+}
+
+TDumpResult TDumpClient::DumpDatabase(const TString& database, const TString& fsPath) {
+ try {
+ NBackup::SetLog(Log);
+ NBackup::BackupDatabase(Driver, database, fsPath);
+ return Result<TDumpResult>();
+ } catch (NBackup::TYdbErrorException& e) {
+ return TDumpResult(std::move(e.Status));
+ } catch (const yexception& e) {
+ return Result<TDumpResult>(EStatus::INTERNAL_ERROR, e.what());
+ }
+}
+
} // NDump
} // NYdb
diff --git a/ydb/public/lib/ydb_cli/dump/dump_impl.h b/ydb/public/lib/ydb_cli/dump/dump_impl.h
index 0ea0b835bd..287e5a48bd 100644
--- a/ydb/public/lib/ydb_cli/dump/dump_impl.h
+++ b/ydb/public/lib/ydb_cli/dump/dump_impl.h
@@ -11,6 +11,10 @@ public:
TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {});
+ TDumpResult DumpCluster(const TString& fsPath);
+
+ TDumpResult DumpDatabase(const TString& database, const TString& fsPath);
+
private:
const TDriver& Driver;
std::shared_ptr<TLog> Log;
diff --git a/ydb/public/lib/ydb_cli/dump/files/files.cpp b/ydb/public/lib/ydb_cli/dump/files/files.cpp
index 90b6074416..496909c623 100644
--- a/ydb/public/lib/ydb_cli/dump/files/files.cpp
+++ b/ydb/public/lib/ydb_cli/dump/files/files.cpp
@@ -14,6 +14,10 @@ enum EFilesType {
INCOMPLETE,
EMPTY,
CREATE_VIEW,
+ DATABASE,
+ CREATE_USER,
+ CREATE_GROUP,
+ ALTER_GROUP,
};
static constexpr TFileInfo FILES_INFO[] = {
@@ -28,6 +32,10 @@ static constexpr TFileInfo FILES_INFO[] = {
{"incomplete", "incomplete"},
{"empty_dir", "empty_dir"},
{"create_view.sql", "view"},
+ {"database.pb", "database description"},
+ {"create_user.sql", "users"},
+ {"create_group.sql", "groups"},
+ {"alter_group.sql", "group members"},
};
const TFileInfo& TableScheme() {
@@ -74,4 +82,20 @@ const TFileInfo& CreateView() {
return FILES_INFO[CREATE_VIEW];
}
+const TFileInfo& Database() {
+ return FILES_INFO[DATABASE];
+}
+
+const TFileInfo& CreateUser() {
+ return FILES_INFO[CREATE_USER];
+}
+
+const TFileInfo& CreateGroup() {
+ return FILES_INFO[CREATE_GROUP];
+}
+
+const TFileInfo& AlterGroup() {
+ return FILES_INFO[ALTER_GROUP];
+}
+
} // NYdb::NDump::NFiles
diff --git a/ydb/public/lib/ydb_cli/dump/files/files.h b/ydb/public/lib/ydb_cli/dump/files/files.h
index 3e65618724..6b0fdca158 100644
--- a/ydb/public/lib/ydb_cli/dump/files/files.h
+++ b/ydb/public/lib/ydb_cli/dump/files/files.h
@@ -18,5 +18,9 @@ const TFileInfo& IncompleteData();
const TFileInfo& Incomplete();
const TFileInfo& Empty();
const TFileInfo& CreateView();
+const TFileInfo& Database();
+const TFileInfo& CreateUser();
+const TFileInfo& CreateGroup();
+const TFileInfo& AlterGroup();
} // NYdb::NDump:NFiles
diff --git a/ydb/public/lib/ydb_cli/dump/util/util.cpp b/ydb/public/lib/ydb_cli/dump/util/util.cpp
index 48e8be3b20..2e35ac36ca 100644
--- a/ydb/public/lib/ydb_cli/dump/util/util.cpp
+++ b/ydb/public/lib/ydb_cli/dump/util/util.cpp
@@ -6,6 +6,7 @@ namespace NYdb::NDump {
using namespace NScheme;
using namespace NTable;
+using namespace NCms;
TStatus DescribeTable(TTableClient& tableClient, const TString& path, TMaybe<TTableDescription>& out) {
auto func = [&path, &out](TSession session) {
@@ -40,4 +41,22 @@ TStatus ModifyPermissions(TSchemeClient& schemeClient, const TString& path, cons
});
}
+TListDirectoryResult ListDirectory(TSchemeClient& schemeClient, const TString& path, const TListDirectorySettings& settings) {
+ return NConsoleClient::RetryFunction([&]() -> TListDirectoryResult {
+ return schemeClient.ListDirectory(path, settings).ExtractValueSync();
+ });
+}
+
+TListDatabasesResult ListDatabases(TCmsClient& cmsClient, const TListDatabasesSettings& settings) {
+ return NConsoleClient::RetryFunction([&]() -> TListDatabasesResult {
+ return cmsClient.ListDatabases(settings).ExtractValueSync();
+ });
+}
+
+TGetDatabaseStatusResult GetDatabaseStatus(TCmsClient& cmsClient, const std::string& path, const TGetDatabaseStatusSettings& settings) {
+ return NConsoleClient::RetryFunction([&]() -> TGetDatabaseStatusResult {
+ return cmsClient.GetDatabaseStatus(path, settings).ExtractValueSync();
+ });
+}
+
}
diff --git a/ydb/public/lib/ydb_cli/dump/util/util.h b/ydb/public/lib/ydb_cli/dump/util/util.h
index 8d9fef2310..25c6da5a17 100644
--- a/ydb/public/lib/ydb_cli/dump/util/util.h
+++ b/ydb/public/lib/ydb_cli/dump/util/util.h
@@ -1,8 +1,9 @@
#pragma once
-#include <ydb-cpp-sdk/client/types/status/status.h>
+#include <ydb-cpp-sdk/client/cms/cms.h>
#include <ydb-cpp-sdk/client/scheme/scheme.h>
#include <ydb-cpp-sdk/client/table/table.h>
+#include <ydb-cpp-sdk/client/types/status/status.h>
#include <util/generic/maybe.h>
#include <util/string/builder.h>
@@ -55,4 +56,19 @@ TStatus ModifyPermissions(
NScheme::TSchemeClient& schemeClient,
const TString& path,
const NScheme::TModifyPermissionsSettings& settings = {});
-}
+
+NScheme::TListDirectoryResult ListDirectory(
+ NScheme::TSchemeClient& schemeClient,
+ const TString& path,
+ const NScheme::TListDirectorySettings& settings = {});
+
+NCms::TListDatabasesResult ListDatabases(
+ NCms::TCmsClient& cmsClient,
+ const NCms::TListDatabasesSettings& settings = {});
+
+NCms::TGetDatabaseStatusResult GetDatabaseStatus(
+ NCms::TCmsClient& cmsClient,
+ const std::string& path,
+ const NCms::TGetDatabaseStatusSettings& settings = {});
+
+} // namespace NYDB::NDump
diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h
index 02e5f11257..15adcfac40 100644
--- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h
+++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h
@@ -3,6 +3,7 @@
#include <ydb-cpp-sdk/client/driver/driver.h>
namespace Ydb::Cms {
+ class CreateDatabaseRequest;
class ListDatabasesResult;
class GetDatabaseStatusResult;
@@ -34,9 +35,7 @@ private:
using TAsyncListDatabasesResult = NThreading::TFuture<TListDatabasesResult>;
-struct TGetDatabaseStatusSettings : public TOperationRequestSettings<TGetDatabaseStatusSettings> {
- FLUENT_SETTING(std::string, Path);
-};
+struct TGetDatabaseStatusSettings : public TOperationRequestSettings<TGetDatabaseStatusSettings> {};
enum class EState {
StateUnspecified = 0,
@@ -165,6 +164,9 @@ public:
const TDatabaseQuotas& GetDatabaseQuotas() const;
const TScaleRecommenderPolicies& GetScaleRecommenderPolicies() const;
+ // Fills CreateDatabaseRequest proto from this database status
+ void SerializeTo(Ydb::Cms::CreateDatabaseRequest& request) const;
+
private:
std::string Path_;
EState State_;
@@ -184,8 +186,9 @@ public:
explicit TCmsClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings());
TAsyncListDatabasesResult ListDatabases(const TListDatabasesSettings& settings = TListDatabasesSettings());
- TAsyncGetDatabaseStatusResult GetDatabaseStatus(const TGetDatabaseStatusSettings& settings = TGetDatabaseStatusSettings());
-
+ TAsyncGetDatabaseStatusResult GetDatabaseStatus(const std::string& path,
+ const TGetDatabaseStatusSettings& settings = TGetDatabaseStatusSettings());
+
private:
class TImpl;
std::shared_ptr<TImpl> Impl_;
diff --git a/ydb/public/sdk/cpp/src/client/cms/cms.cpp b/ydb/public/sdk/cpp/src/client/cms/cms.cpp
index da178521d9..3951f95696 100644
--- a/ydb/public/sdk/cpp/src/client/cms/cms.cpp
+++ b/ydb/public/sdk/cpp/src/client/cms/cms.cpp
@@ -174,6 +174,71 @@ const TScaleRecommenderPolicies& TGetDatabaseStatusResult::GetScaleRecommenderPo
return ScaleRecommenderPolicies_;
}
+void TGetDatabaseStatusResult::SerializeTo(Ydb::Cms::CreateDatabaseRequest& request) const {
+ request.set_path(Path_);
+ if (std::holds_alternative<NCms::TResources>(ResourcesKind_)) {
+ const auto& resources = std::get<NCms::TResources>(ResourcesKind_);
+ for (const auto& storageUnit : resources.StorageUnits) {
+ auto* protoUnit = request.mutable_resources()->add_storage_units();
+ protoUnit->set_unit_kind(storageUnit.UnitKind);
+ protoUnit->set_count(storageUnit.Count);
+ }
+ for (const auto& computationalUnit : resources.ComputationalUnits) {
+ auto* protoUnit = request.mutable_resources()->add_computational_units();
+ protoUnit->set_unit_kind(computationalUnit.UnitKind);
+ protoUnit->set_count(computationalUnit.Count);
+ protoUnit->set_availability_zone(computationalUnit.AvailabilityZone);
+ }
+ } else if (std::holds_alternative<NCms::TSharedResources>(ResourcesKind_)) {
+ const auto& resources = std::get<NCms::TSharedResources>(ResourcesKind_);
+ for (const auto& storageUnit : resources.StorageUnits) {
+ auto* protoUnit = request.mutable_shared_resources()->add_storage_units();
+ protoUnit->set_unit_kind(storageUnit.UnitKind);
+ protoUnit->set_count(storageUnit.Count);
+ }
+ for (const auto& computationalUnit : resources.ComputationalUnits) {
+ auto* protoUnit = request.mutable_shared_resources()->add_computational_units();
+ protoUnit->set_unit_kind(computationalUnit.UnitKind);
+ protoUnit->set_count(computationalUnit.Count);
+ protoUnit->set_availability_zone(computationalUnit.AvailabilityZone);
+ }
+ } else if (std::holds_alternative<NCms::TServerlessResources>(ResourcesKind_)) {
+ const auto& resources = std::get<NCms::TServerlessResources>(ResourcesKind_);
+ request.mutable_serverless_resources()->set_shared_database_path(resources.SharedDatabasePath);
+ }
+
+ for (const auto& quota : SchemaOperationQuotas_.LeakyBucketQuotas) {
+ auto protoQuota = request.mutable_schema_operation_quotas()->add_leaky_bucket_quotas();
+ protoQuota->set_bucket_seconds(quota.BucketSeconds);
+ protoQuota->set_bucket_size(quota.BucketSize);
+ }
+
+ request.mutable_database_quotas()->set_data_size_hard_quota(DatabaseQuotas_.DataSizeHardQuota);
+ request.mutable_database_quotas()->set_data_size_soft_quota(DatabaseQuotas_.DataSizeSoftQuota);
+ request.mutable_database_quotas()->set_data_stream_shards_quota(DatabaseQuotas_.DataStreamShardsQuota);
+ request.mutable_database_quotas()->set_data_stream_reserved_storage_quota(DatabaseQuotas_.DataStreamReservedStorageQuota);
+ request.mutable_database_quotas()->set_ttl_min_run_internal_seconds(DatabaseQuotas_.TtlMinRunInternalSeconds);
+
+ for (const auto& quota : DatabaseQuotas_.StorageQuotas) {
+ auto protoQuota = request.mutable_database_quotas()->add_storage_quotas();
+ protoQuota->set_unit_kind(quota.UnitKind);
+ protoQuota->set_data_size_hard_quota(quota.DataSizeHardQuota);
+ protoQuota->set_data_size_soft_quota(quota.DataSizeSoftQuota);
+ }
+
+ for (const auto& policy : ScaleRecommenderPolicies_.Policies) {
+ auto* protoPolicy = request.mutable_scale_recommender_policies()->add_policies();
+ if (std::holds_alternative<NCms::TTargetTrackingPolicy>(policy.Policy)) {
+ const auto& targetTracking = std::get<NCms::TTargetTrackingPolicy>(policy.Policy);
+ auto* protoTargetTracking = protoPolicy->mutable_target_tracking_policy();
+ if (std::holds_alternative<NCms::TTargetTrackingPolicy::TAverageCpuUtilizationPercent>(targetTracking.Target)) {
+ const auto& target = std::get<NCms::TTargetTrackingPolicy::TAverageCpuUtilizationPercent>(targetTracking.Target);
+ protoTargetTracking->set_average_cpu_utilization_percent(target);
+ }
+ }
+ }
+}
+
class TCmsClient::TImpl : public TClientImplCommon<TCmsClient::TImpl> {
public:
TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings)
@@ -206,9 +271,9 @@ public:
return promise.GetFuture();
}
- TAsyncGetDatabaseStatusResult GetDatabaseStatus(const TGetDatabaseStatusSettings& settings) {
+ TAsyncGetDatabaseStatusResult GetDatabaseStatus(const std::string& path, const TGetDatabaseStatusSettings& settings) {
Ydb::Cms::GetDatabaseStatusRequest request;
- request.set_path(settings.Path_);
+ request.set_path(path);
auto promise = NThreading::NewPromise<TGetDatabaseStatusResult>();
@@ -242,8 +307,11 @@ TAsyncListDatabasesResult TCmsClient::ListDatabases(const TListDatabasesSettings
return Impl_->ListDatabases(settings);
}
-TAsyncGetDatabaseStatusResult TCmsClient::GetDatabaseStatus(const TGetDatabaseStatusSettings& settings) {
- return Impl_->GetDatabaseStatus(settings);
+TAsyncGetDatabaseStatusResult TCmsClient::GetDatabaseStatus(
+ const std::string& path,
+ const TGetDatabaseStatusSettings& settings)
+{
+ return Impl_->GetDatabaseStatus(path, settings);
}
} // namespace NYdb::NCms
diff --git a/ydb/tests/functional/ydb_cli/test_ydb_backup.py b/ydb/tests/functional/ydb_cli/test_ydb_backup.py
index bf5ee44df4..301b9bd1be 100644
--- a/ydb/tests/functional/ydb_cli/test_ydb_backup.py
+++ b/ydb/tests/functional/ydb_cli/test_ydb_backup.py
@@ -4,9 +4,10 @@ from ydb.tests.library.harness.kikimr_runner import KiKiMR
from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
from ydb.tests.oss.ydb_sdk_import import ydb
-from hamcrest import assert_that, is_, is_not, contains_inanyorder, has_items
-import os
+from hamcrest import assert_that, is_, is_not, contains_inanyorder, has_items, equal_to
+import enum
import logging
+import os
import pytest
import yatest
@@ -186,18 +187,25 @@ def is_permissions_the_same(scheme_client, path_left, path_right):
return True
-def list_all_dirs(prefix, path=""):
+@enum.unique
+class ListMode(enum.IntEnum):
+ DIRS = 0,
+ FILES = 1,
+
+
+def fs_recursive_list(prefix, mode=ListMode.DIRS, path=""):
paths = []
full_path = os.path.join(prefix, path)
logger.debug("prefix# " + prefix + " path# " + path)
for item in os.listdir(full_path):
item_path = os.path.join(full_path, item)
if os.path.isdir(item_path):
- paths.append(os.path.join(path, item))
- paths += list_all_dirs(prefix, os.path.join(path, item))
- else:
- # don't list regular files
- pass
+ if mode == ListMode.DIRS:
+ paths.append(os.path.join(path, item))
+ paths += fs_recursive_list(prefix, mode, os.path.join(path, item))
+ elif os.path.isfile(item_path):
+ if mode == ListMode.FILES:
+ paths.append(os.path.join(path, item))
return paths
@@ -244,12 +252,12 @@ class BaseTestBackupInFiles(object):
)
logger.debug("std_out:\n" + execution.std_out.decode('utf-8'))
- list_all_dirs(backup_files_dir)
- logger.debug("list_all_dirs(backup_files_dir)# " + str(list_all_dirs(backup_files_dir)))
+ fs_recursive_list(backup_files_dir)
+ logger.debug("fs_recursive_list(backup_files_dir)# " + str(fs_recursive_list(backup_files_dir)))
logger.debug("expected_dirs# " + str(expected_dirs))
assert_that(
- list_all_dirs(backup_files_dir),
+ fs_recursive_list(backup_files_dir),
has_items(*expected_dirs)
)
@@ -1286,3 +1294,133 @@ class TestRestoreNoData(BaseTestBackupInFiles):
is_data_the_same(session, "/Root/folder/table", "/Root/restored/table"),
is_(False)
)
+
+
+class BaseTestClusterBackupInFiles(object):
+ @classmethod
+ def setup_class(cls):
+ cls.cluster = KiKiMR(KikimrConfigGenerator(extra_feature_flags=["enable_resource_pools"]))
+ cls.cluster.start()
+
+ cls.root_dir = "/Root"
+ cls.database = os.path.join(cls.root_dir, "db1")
+
+ cls.cluster.create_database(
+ cls.database,
+ storage_pool_units_count={
+ 'hdd': 1
+ },
+ timeout_seconds=100
+ )
+
+ cls.database_nodes = cls.cluster.register_and_start_slots(cls.database, count=3)
+ cls.cluster.wait_tenant_up(cls.database)
+
+ driver_config = ydb.DriverConfig(
+ database=cls.database,
+ endpoint="%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port))
+ cls.driver = ydb.Driver(driver_config)
+ cls.driver.wait(timeout=4)
+
+ @classmethod
+ def teardown_class(cls):
+ cls.cluster.unregister_and_stop_slots(cls.database_nodes)
+ cls.cluster.stop()
+
+ @pytest.fixture(autouse=True, scope='class')
+ @classmethod
+ def set_test_name(cls, request):
+ cls.test_name = request.node.name
+
+ @classmethod
+ def create_backup(cls, command, expected_files, additional_args=[]):
+ backup_files_dir = output_path(cls.test_name, "backup_files_dir")
+ execution = yatest.common.execute(
+ [
+ backup_bin(),
+ "--verbose",
+ "--assume-yes",
+ "--endpoint", "grpc://localhost:%d" % cls.cluster.nodes[1].grpc_port,
+ ]
+ + command
+ + ["--output", backup_files_dir]
+ + additional_args
+ )
+
+ list_result = fs_recursive_list(backup_files_dir, ListMode.FILES)
+
+ logger.debug("std_out:\n" + execution.std_out.decode('utf-8'))
+ logger.debug("fs_recursive_list(backup_files_dir)# " + str(list_result))
+ logger.debug("expected_files# " + str(expected_files))
+
+ assert_that(
+ len(list_result),
+ equal_to(len(expected_files))
+ )
+
+ assert_that(
+ list_result,
+ has_items(*expected_files)
+ )
+
+ @classmethod
+ def create_cluster_backup(cls, expected_files, additional_args=[]):
+ cls.create_backup(
+ [
+ "admin", "cluster", "dump",
+ ],
+ expected_files,
+ additional_args
+ )
+
+ @classmethod
+ def create_database_backup(cls, expected_files, additional_args=[]):
+ cls.create_backup(
+ [
+ "--database", cls.database,
+ "admin", "database", "dump",
+ ],
+ expected_files,
+ additional_args
+ )
+
+
+class TestClusterBackup(BaseTestClusterBackupInFiles):
+ def test_cluster_backup(self):
+ session = self.driver.table_client.session().create()
+ create_table_with_data(session, "db1/table")
+
+ self.create_cluster_backup(expected_files=[
+ # cluster metadata
+ "permissions.pb",
+ "create_user.sql",
+ "create_group.sql",
+ "alter_group.sql",
+
+ # database metadata
+ "Root/db1/database.pb",
+ "Root/db1/permissions.pb",
+ "Root/db1/create_user.sql",
+ "Root/db1/create_group.sql",
+ "Root/db1/alter_group.sql",
+ ])
+
+
+class TestDatabaseBackup(BaseTestClusterBackupInFiles):
+ def test_database_backup(self):
+ session = self.driver.table_client.session().create()
+ create_table_with_data(session, "db1/table")
+
+ self.create_database_backup(expected_files=[
+ # database metadata
+ "database.pb",
+ "permissions.pb",
+ "create_user.sql",
+ "create_group.sql",
+ "alter_group.sql",
+
+ # database table
+ "table/scheme.pb",
+ "table/permissions.pb",
+ "table/data_00.csv",
+ ])