diff options
author | Ilia Shakhov <pixcc@ydb.tech> | 2025-02-10 21:16:55 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2025-02-10 21:16:55 +0300 |
commit | 80d32f2f8ea4092e4cff97e64f6a149621a8c956 (patch) | |
tree | 400e9ee83a4853c3d069c52afa9b4ab6bcdf369f | |
parent | 4878da3dd9b3b683b5420c7dbb403fe035ff74cd (diff) | |
download | ydb-80d32f2f8ea4092e4cff97e64f6a149621a8c956.tar.gz |
Add cluster & database dump to YDB CLI (#14306)
20 files changed, 803 insertions, 36 deletions
diff --git a/ydb/library/backup/backup.cpp b/ydb/library/backup/backup.cpp index 6c014a2874..7024072ac2 100644 --- a/ydb/library/backup/backup.cpp +++ b/ydb/library/backup/backup.cpp @@ -2,7 +2,16 @@ #include "db_iterator.h" #include "util.h" +#include <ydb-cpp-sdk/client/cms/cms.h> +#include <ydb-cpp-sdk/client/draft/ydb_view.h> +#include <ydb-cpp-sdk/client/driver/driver.h> +#include <ydb-cpp-sdk/client/proto/accessor.h> +#include <ydb-cpp-sdk/client/result/result.h> +#include <ydb-cpp-sdk/client/table/table.h> +#include <ydb-cpp-sdk/client/topic/client.h> +#include <ydb-cpp-sdk/client/value/value.h> #include <ydb/public/api/protos/draft/ydb_view.pb.h> +#include <ydb/public/api/protos/ydb_cms.pb.h> #include <ydb/public/api/protos/ydb_rate_limiter.pb.h> #include <ydb/public/api/protos/ydb_table.pb.h> #include <ydb/public/lib/ydb_cli/common/recursive_remove.h> @@ -676,9 +685,9 @@ void BackupCoordinationNode(TDriver driver, const TString& dbPath, const TFsPath void CreateClusterDirectory(const TDriver& driver, const TString& path, bool rootBackupDir = false) { if (rootBackupDir) { - LOG_I("Create temporary directory " << path.Quote()); + LOG_I("Create temporary directory " << path.Quote() << " in database"); } else { - LOG_D("Create directory " << path.Quote()); + LOG_D("Create directory " << path.Quote() << " in database"); } NScheme::TSchemeClient client(driver); TStatus status = client.MakeDirectory(path).GetValueSync(); @@ -693,7 +702,7 @@ void RemoveClusterDirectory(const TDriver& driver, const TString& path) { } void RemoveClusterDirectoryRecursive(const TDriver& driver, const TString& path) { - LOG_I("Remove temporary directory " << path.Quote()); + LOG_I("Remove temporary directory " << path.Quote() << " in database"); NScheme::TSchemeClient schemeClient(driver); NTable::TTableClient tableClient(driver); TStatus status = NConsoleClient::RemoveDirectoryRecursive(schemeClient, tableClient, path, {}, true, false); @@ -842,6 +851,281 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists(); } +namespace { + +NCms::TListDatabasesResult ListDatabases(TDriver driver) { + NCms::TCmsClient client(driver); + + auto status = NDump::ListDatabases(client); + VerifyStatus(status); + + return status; +} + +NCms::TGetDatabaseStatusResult GetDatabaseStatus(TDriver driver, const std::string& path) { + NCms::TCmsClient client(driver); + + auto status = NDump::GetDatabaseStatus(client, path); + VerifyStatus(status); + + return status; +} + +bool IsNotLowerAlphaNum(char c) { + if (isalnum(c)) { + if (isalpha(c)) { + return !islower(c); + } + + return false; + } + + return true; +} + +bool IsValidSid(const std::string& sid) { + return std::find_if(sid.begin(), sid.end(), IsNotLowerAlphaNum) == sid.end(); +} + +struct TAdmins { + TString GroupSid; + THashSet<TString> UserSids; +}; + +TAdmins FindAdmins(TDriver driver, const TString& dbPath) { + THashSet<TString> adminUserSids; + + auto entry = DescribePath(driver, dbPath); + + NYdb::NTable::TTableClient client(driver); + auto query = Sprintf("SELECT * FROM `%s/.sys/auth_group_members`", dbPath.c_str()); + auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(); + + std::vector<TResultSet> resultSets; + TStatus status = client.RetryOperationSync([&](NTable::TSession session) { + auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync(); + VerifyStatus(result); + resultSets = result.GetResultSets(); + return result; + }); + VerifyStatus(status); + + TStringStream alterGroupQuery; + for (const auto& resultSet : resultSets) { + TResultSetParser parser(resultSet); + while (parser.TryNextRow()) { + auto groupSidValue = parser.GetValue("GroupSid"); + auto memberSidValue = parser.GetValue("MemberSid"); + + const auto& groupSid = groupSidValue.GetProto().text_value(); + const auto& memberSid = memberSidValue.GetProto().text_value(); + + if (groupSid == entry.Owner) { + adminUserSids.insert(memberSid); + } + } + } + + return { TString(entry.Owner), adminUserSids }; +} + +struct TBackupDatabaseSettings { + bool WithRegularUsers = false; + bool WithContent = false; + TString TemporalBackupPostfix; +}; + +void BackupUsers(TDriver driver, const TString& dbPath, const TFsPath& folderPath, const THashSet<TString>& filter = {}) { + NYdb::NTable::TTableClient client(driver); + auto query = Sprintf("SELECT * FROM `%s/.sys/auth_users`", dbPath.c_str()); + auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(); + + std::vector<TResultSet> resultSets; + TStatus status = client.RetryOperationSync([&](NTable::TSession session) { + auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync(); + VerifyStatus(result); + resultSets = result.GetResultSets(); + return result; + }); + VerifyStatus(status); + + TStringStream createUserQuery; + for (const auto& resultSet : resultSets) { + TResultSetParser parser(resultSet); + while (parser.TryNextRow()) { + auto sidValue = parser.GetValue("Sid"); + auto passwordValue = parser.GetValue("PasswordHash"); + auto isEnabledValue = parser.GetValue("IsEnabled"); + + const auto& sid = sidValue.GetProto().text_value(); + const auto& password = passwordValue.GetProto().text_value(); + bool isEnabled = isEnabledValue.GetProto().bool_value(); + + if (!filter.empty() && !filter.contains(sid)) { + continue; + } + + // Some SIDs may be created through configuration that bypasses this checks + if (IsValidSid(sid)) { + createUserQuery << Sprintf("CREATE USER `%s` HASH '%s';\n", sid.c_str(), password.c_str()); + } + + if (!isEnabled) { + createUserQuery << Sprintf("ALTER USER `%s` NOLOGIN;\n", sid.c_str()); + } + } + } + + WriteCreationQueryToFile(createUserQuery.Str(), folderPath, NDump::NFiles::CreateUser()); +} + +void BackupGroups(TDriver driver, const TString& dbPath, const TFsPath& folderPath, const THashSet<TString>& filter = {}) { + NYdb::NTable::TTableClient client(driver); + auto query = Sprintf("SELECT * FROM `%s/.sys/auth_groups`", dbPath.c_str()); + auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(); + + std::vector<TResultSet> resultSets; + TStatus status = client.RetryOperationSync([&](NTable::TSession session) { + auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync(); + VerifyStatus(result); + resultSets = result.GetResultSets(); + return result; + }); + VerifyStatus(status); + + TStringStream createGroupQuery; + for (const auto& resultSet : resultSets) { + TResultSetParser parser(resultSet); + while (parser.TryNextRow()) { + auto sidValue = parser.GetValue("Sid"); + const auto& sid = sidValue.GetProto().text_value(); + + if (!filter.empty() && !filter.contains(sid)) { + continue; + } + + // Some SIDs may be created through configuration that bypasses this checks + if (IsValidSid(sid)) { + createGroupQuery << Sprintf("CREATE GROUP `%s`;\n", sid.c_str()); + } + } + } + + WriteCreationQueryToFile(createGroupQuery.Str(), folderPath, NDump::NFiles::CreateGroup()); +} + +void BackupGroupMembers(TDriver driver, const TString& dbPath, const TFsPath& folderPath, const THashSet<TString>& filterGroups = {}) { + NYdb::NTable::TTableClient client(driver); + auto query = Sprintf("SELECT * FROM `%s/.sys/auth_group_members`", dbPath.c_str()); + auto settings = NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(); + + std::vector<TResultSet> resultSets; + TStatus status = client.RetryOperationSync([&](NTable::TSession session) { + auto result = session.ExecuteDataQuery(query, settings).ExtractValueSync(); + VerifyStatus(result); + resultSets = result.GetResultSets(); + return result; + }); + VerifyStatus(status); + + TStringStream alterGroupQuery; + for (const auto& resultSet : resultSets) { + TResultSetParser parser(resultSet); + while (parser.TryNextRow()) { + auto groupSidValue = parser.GetValue("GroupSid"); + auto memberSidValue = parser.GetValue("MemberSid"); + + const auto& groupSid = groupSidValue.GetProto().text_value(); + const auto& memberSid = memberSidValue.GetProto().text_value(); + + if (!filterGroups.empty() && !filterGroups.contains(groupSid)) { + continue; + } + + alterGroupQuery << Sprintf("ALTER GROUP `%s` ADD USER `%s`;\n", groupSid.c_str(), memberSid.c_str()); + } + } + + WriteCreationQueryToFile(alterGroupQuery.Str(), folderPath, NDump::NFiles::AlterGroup()); +} + +void BackupDatabaseImpl(TDriver driver, const TString& dbPath, const TFsPath& folderPath, TBackupDatabaseSettings settings) { + LOG_I("Backup database " << dbPath.Quote() << " to " << folderPath.GetPath().Quote()); + folderPath.MkDirs(); + + auto status = GetDatabaseStatus(driver, dbPath); + Ydb::Cms::CreateDatabaseRequest proto; + status.SerializeTo(proto); + WriteProtoToFile(proto, folderPath, NDump::NFiles::Database()); + + if (!settings.WithRegularUsers) { + TAdmins admins = FindAdmins(driver, dbPath); + BackupUsers(driver, dbPath, folderPath, admins.UserSids); + BackupGroups(driver, dbPath, folderPath, { admins.GroupSid }); + BackupGroupMembers(driver, dbPath, folderPath, { admins.GroupSid }); + } else { + BackupUsers(driver, dbPath, folderPath); + BackupGroups(driver, dbPath, folderPath); + BackupGroupMembers(driver, dbPath, folderPath); + } + + BackupPermissions(driver, dbPath, "", folderPath); + if (settings.WithContent) { + // full path to temporal directory in database + TString tmpDbFolder; + try { + tmpDbFolder = JoinDatabasePath(dbPath, "~" + settings.TemporalBackupPostfix); + CreateClusterDirectory(driver, tmpDbFolder, true); + + NYql::TIssues issues; + BackupFolderImpl( + driver, + dbPath, + tmpDbFolder, + folderPath, + /* exclusionPatterns */ {}, + /* schemaOnly */ false, + /* useConsistentCopyTable */ true, + /* avoidCopy */ false, + /* preservePoolKinds */ false, + /* ordered */ false, + issues + ); + + if (issues) { + Cerr << issues.ToString(); + } + } catch (...) { + RemoveClusterDirectoryRecursive(driver, tmpDbFolder); + folderPath.ForceDelete(); + throw; + } + RemoveClusterDirectoryRecursive(driver, tmpDbFolder); + } +} + +TString FindClusterRootPath(TDriver driver) { + NScheme::TSchemeClient client(driver); + auto status = NDump::ListDirectory(client, "/"); + VerifyStatus(status); + + Y_ENSURE(status.GetChildren().size() == 1, "Exactly one cluster root expected, found: " << JoinSeq(", ", status.GetChildren())); + return "/" + status.GetChildren().begin()->Name; +} + +void BackupClusterRoot(TDriver driver, const TFsPath& folderPath) { + TString rootPath = FindClusterRootPath(driver); + + LOG_I("Backup cluster root " << rootPath.Quote() << " to " << folderPath); + + BackupUsers(driver, rootPath, folderPath); + BackupGroups(driver, rootPath, folderPath); + BackupGroupMembers(driver, rootPath, folderPath); + BackupPermissions(driver, rootPath, "", folderPath); +} + +} // anonymous namespace + void CheckedCreateBackupFolder(const TFsPath& folderPath) { const bool exists = folderPath.Exists(); if (exists) { @@ -906,4 +1190,67 @@ void BackupFolder(const TDriver& driver, const TString& database, const TString& LOG_I("Backup completed successfully"); } +void BackupDatabase(const TDriver& driver, const TString& database, TFsPath folderPath) { + TString temporalBackupPostfix = CreateTemporalBackupName(); + if (!folderPath) { + folderPath = temporalBackupPostfix; + } + CheckedCreateBackupFolder(folderPath); + + try { + NYql::TIssues issues; + TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways); + + BackupDatabaseImpl(driver, database, folderPath, { + .WithRegularUsers = true, + .WithContent = true, + .TemporalBackupPostfix = temporalBackupPostfix, + }); + + folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists(); + if (issues) { + Cerr << issues.ToString(); + } + } catch (...) { + LOG_E("Backup failed"); + folderPath.ForceDelete(); + throw; + } + LOG_I("Backup database " << database.Quote() << " is completed successfully"); +} + +void BackupCluster(const TDriver& driver, TFsPath folderPath) { + TString temporalBackupPostfix = CreateTemporalBackupName(); + if (!folderPath) { + folderPath = temporalBackupPostfix; + } + CheckedCreateBackupFolder(folderPath); + + LOG_I("Backup cluster to " << folderPath.GetPath().Quote()); + + try { + NYql::TIssues issues; + TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways); + + BackupClusterRoot(driver, folderPath); + auto databases = ListDatabases(driver); + for (const auto& database : databases.GetPaths()) { + BackupDatabaseImpl(driver, TString(database), folderPath.Child("." + database), { + .WithRegularUsers = false, + .WithContent = false, + }); + } + + folderPath.Child(NDump::NFiles::Incomplete().FileName).DeleteIfExists(); + if (issues) { + Cerr << issues.ToString(); + } + } catch (...) { + LOG_E("Backup failed"); + folderPath.ForceDelete(); + throw; + } + LOG_I("Backup cluster is completed successfully"); +} + } // NYdb::NBackup diff --git a/ydb/library/backup/backup.h b/ydb/library/backup/backup.h index dfd96b40b3..c93c20d884 100644 --- a/ydb/library/backup/backup.h +++ b/ydb/library/backup/backup.h @@ -46,6 +46,9 @@ void BackupFolder( bool preservePoolKinds = false, bool ordered = false); +void BackupCluster(const TDriver& driver, TFsPath folderPath); +void BackupDatabase(const TDriver& driver, const TString& database, TFsPath folderPath); + // For unit-tests only TMaybe<TValue> ProcessResultSet( TStringStream& ss, diff --git a/ydb/library/backup/ya.make b/ydb/library/backup/ya.make index ee77236af7..0507feabcc 100644 --- a/ydb/library/backup/ya.make +++ b/ydb/library/backup/ya.make @@ -12,6 +12,7 @@ PEERDIR( ydb/public/lib/ydb_cli/dump/util ydb/public/lib/yson_value ydb/public/lib/ydb_cli/dump/files + ydb/public/sdk/cpp/src/client/cms ydb/public/sdk/cpp/src/client/coordination ydb/public/sdk/cpp/src/client/draft ydb/public/sdk/cpp/src/client/driver diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp index 6c7beb1f13..21208491ef 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp @@ -5,6 +5,11 @@ #include "ydb_cluster.h" #include <ydb/public/lib/ydb_cli/common/command_utils.h> +#include <ydb/public/lib/ydb_cli/dump/dump.h> + +#define INCLUDE_YDB_INTERNAL_H +#include <ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h> +#undef INCLUDE_YDB_INTERNAL_H namespace NYdb { namespace NConsoleClient { @@ -24,9 +29,39 @@ public: : TClientCommandTree("database", {}, "Database-wide administration") { AddCommand(std::make_unique<NDynamicConfig::TCommandConfig>()); + AddCommand(std::make_unique<TCommandDatabaseDump>()); } }; +TCommandDatabaseDump::TCommandDatabaseDump() + : TYdbCommand("dump", {}, "Dump database into local directory") +{} + +void TCommandDatabaseDump::Config(TConfig& config) { + TYdbCommand::Config(config); + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption('o', "output", "Path in a local filesystem to a directory to place dump into." + " Directory should either not exist or be empty." + " If not specified, the dump is placed in the directory backup_YYYYYYMMDDDThhmmss.") + .RequiredArgument("PATH") + .StoreResult(&FilePath); +} + +void TCommandDatabaseDump::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandDatabaseDump::Run(TConfig& config) { + auto log = std::make_shared<TLog>(CreateLogBackend("cerr", TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel))); + log->SetFormatter(GetPrefixLogFormatter("")); + + NDump::TClient client(CreateDriver(config), std::move(log)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.DumpDatabase(config.Database, FilePath)); + + return EXIT_SUCCESS; +} + TCommandAdmin::TCommandAdmin() : TClientCommandTree("admin", {}, "Administrative cluster operations") { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.h b/ydb/public/lib/ydb_cli/commands/ydb_admin.h index d544544ec9..8e0d65d836 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_admin.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.h @@ -12,5 +12,16 @@ protected: virtual void Config(TConfig& config) override; }; +class TCommandDatabaseDump : public TYdbCommand { +public: + TCommandDatabaseDump(); + void Config(TConfig& config) override; + void Parse(TConfig& config) override; + int Run(TConfig& config) override; + +private: + TString FilePath; +}; + } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index 50c3c3e66b..959873ab88 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -1,8 +1,14 @@ #include "ydb_cluster.h" -#include <ydb-cpp-sdk/client/bsconfig/storage_config.h> #include "ydb_dynamic_config.h" +#include <ydb-cpp-sdk/client/bsconfig/storage_config.h> +#include <ydb/public/lib/ydb_cli/dump/dump.h> + +#define INCLUDE_YDB_INTERNAL_H +#include <ydb/public/sdk/cpp/src/client/impl/ydb_internal/logger/log.h> +#undef INCLUDE_YDB_INTERNAL_H + using namespace NKikimr; namespace NYdb::NConsoleClient::NCluster { @@ -12,6 +18,7 @@ TCommandCluster::TCommandCluster() { AddCommand(std::make_unique<TCommandClusterBootstrap>()); AddCommand(std::make_unique<NDynamicConfig::TCommandConfig>(true)); + AddCommand(std::make_unique<TCommandClusterDump>()); } TCommandClusterBootstrap::TCommandClusterBootstrap() @@ -37,4 +44,34 @@ int TCommandClusterBootstrap::Run(TConfig& config) { return EXIT_SUCCESS; } +TCommandClusterDump::TCommandClusterDump() + : TYdbCommand("dump", {}, "Dump cluster into local directory") +{} + +void TCommandClusterDump::Config(TConfig& config) { + TYdbCommand::Config(config); + config.SetFreeArgsNum(0); + config.AllowEmptyDatabase = true; + + config.Opts->AddLongOption('o', "output", "Path in a local filesystem to a directory to place dump into." + " Directory should either not exist or be empty." + " If not specified, the dump is placed in the directory backup_YYYYYYMMDDDThhmmss.") + .RequiredArgument("PATH") + .StoreResult(&FilePath); +} + +void TCommandClusterDump::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandClusterDump::Run(TConfig& config) { + auto log = std::make_shared<TLog>(CreateLogBackend("cerr", TConfig::VerbosityLevelToELogPriority(config.VerbosityLevel))); + log->SetFormatter(GetPrefixLogFormatter("")); + + NDump::TClient client(CreateDriver(config), std::move(log)); + NStatusHelpers::ThrowOnErrorOrPrintIssues(client.DumpCluster(FilePath)); + + return EXIT_SUCCESS; +} + } // namespace NYdb::NConsoleClient::NCluster diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.h b/ydb/public/lib/ydb_cli/commands/ydb_cluster.h index 6a5339979c..ebe4375fde 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.h @@ -22,4 +22,15 @@ public: int Run(TConfig& config) override; }; +class TCommandClusterDump : public TYdbCommand { +public: + TCommandClusterDump(); + void Config(TConfig& config) override; + void Parse(TConfig& config) override; + int Run(TConfig& config) override; + +private: + TString FilePath; +}; + } // namespace NYdb::NConsoleClient::NCluster diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 5a1b71c537..a4960f999a 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -161,7 +161,7 @@ void TClientCommand::SaveParseResult(TConfig& config) { } void TClientCommand::Prepare(TConfig& config) { - config.ArgsSettings.Reset(new TConfig::TArgSettings()); + config.ArgsSettings = TConfig::TArgSettings(); config.Opts = &Opts; Config(config); CheckForExecutableOptions(config); diff --git a/ydb/public/lib/ydb_cli/common/command.h b/ydb/public/lib/ydb_cli/common/command.h index 412ab9b9b8..abf6ad6bd4 100644 --- a/ydb/public/lib/ydb_cli/common/command.h +++ b/ydb/public/lib/ydb_cli/common/command.h @@ -104,7 +104,7 @@ public: THashSet<TString> ExecutableOptions; bool HasExecutableOptions = false; TString Path; - THolder<TArgSettings> ArgsSettings; + TArgSettings ArgsSettings; TString Address; TString Database; TString CaCerts; @@ -185,18 +185,18 @@ public: } void SetFreeArgsMin(size_t value) { - ArgsSettings->Min.Set(value); + ArgsSettings.Min.Set(value); Opts->SetFreeArgsMin(value); } void SetFreeArgsMax(size_t value) { - ArgsSettings->Max.Set(value); + ArgsSettings.Max.Set(value); Opts->SetFreeArgsMax(value); } void SetFreeArgsNum(size_t minValue, size_t maxValue) { - ArgsSettings->Min.Set(minValue); - ArgsSettings->Max.Set(maxValue); + ArgsSettings.Min.Set(minValue); + ArgsSettings.Max.Set(maxValue); Opts->SetFreeArgsNum(minValue, maxValue); } @@ -209,10 +209,10 @@ public: if (HasHelpCommand() || HasExecutableOptions) { return; } - bool minSet = ArgsSettings->Min.GetIsSet(); - size_t minValue = ArgsSettings->Min.Get(); - bool maxSet = ArgsSettings->Max.GetIsSet(); - size_t maxValue = ArgsSettings->Max.Get(); + bool minSet = ArgsSettings.Min.GetIsSet(); + size_t minValue = ArgsSettings.Min.Get(); + bool maxSet = ArgsSettings.Max.GetIsSet(); + size_t maxValue = ArgsSettings.Max.Get(); bool minFailed = minSet && count < minValue; bool maxFailed = maxSet && count > maxValue; if (minFailed || maxFailed) { diff --git a/ydb/public/lib/ydb_cli/dump/dump.cpp b/ydb/public/lib/ydb_cli/dump/dump.cpp index c4acb28619..cbaa7b41b4 100644 --- a/ydb/public/lib/ydb_cli/dump/dump.cpp +++ b/ydb/public/lib/ydb_cli/dump/dump.cpp @@ -33,6 +33,16 @@ public: return client.Restore(fsPath, dbPath, settings); } + TDumpResult DumpCluster(const TString& fsPath) { + auto client = TDumpClient(Driver, Log); + return client.DumpCluster(fsPath); + } + + TDumpResult DumpDatabase(const TString& database, const TString& fsPath) { + auto client = TDumpClient(Driver, Log); + return client.DumpDatabase(database, fsPath); + } + private: const TDriver Driver; std::shared_ptr<TLog> Log; @@ -67,5 +77,13 @@ TRestoreResult TClient::Restore(const TString& fsPath, const TString& dbPath, co return Impl_->Restore(fsPath, dbPath, settings); } +TDumpResult TClient::DumpCluster(const TString& fsPath) { + return Impl_->DumpCluster(fsPath); +} + +TDumpResult TClient::DumpDatabase(const TString& database, const TString& fsPath) { + return Impl_->DumpDatabase(database, fsPath); +} + } // NDump } // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/dump.h b/ydb/public/lib/ydb_cli/dump/dump.h index 1f85a88230..1c5374c3f0 100644 --- a/ydb/public/lib/ydb_cli/dump/dump.h +++ b/ydb/public/lib/ydb_cli/dump/dump.h @@ -130,6 +130,10 @@ public: TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {}); TRestoreResult Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings = {}); + TDumpResult DumpCluster(const TString& fsPath); + + TDumpResult DumpDatabase(const TString& database, const TString& fsPath); + private: std::shared_ptr<TImpl> Impl_; diff --git a/ydb/public/lib/ydb_cli/dump/dump_impl.cpp b/ydb/public/lib/ydb_cli/dump/dump_impl.cpp index b92342d231..be3eb896eb 100644 --- a/ydb/public/lib/ydb_cli/dump/dump_impl.cpp +++ b/ydb/public/lib/ydb_cli/dump/dump_impl.cpp @@ -36,5 +36,29 @@ TDumpResult TDumpClient::Dump(const TString& dbPath, const TString& fsPath, cons } } +TDumpResult TDumpClient::DumpCluster(const TString& fsPath) { + try { + NBackup::SetLog(Log); + NBackup::BackupCluster(Driver, fsPath); + return Result<TDumpResult>(); + } catch (NBackup::TYdbErrorException& e) { + return TDumpResult(std::move(e.Status)); + } catch (const yexception& e) { + return Result<TDumpResult>(EStatus::INTERNAL_ERROR, e.what()); + } +} + +TDumpResult TDumpClient::DumpDatabase(const TString& database, const TString& fsPath) { + try { + NBackup::SetLog(Log); + NBackup::BackupDatabase(Driver, database, fsPath); + return Result<TDumpResult>(); + } catch (NBackup::TYdbErrorException& e) { + return TDumpResult(std::move(e.Status)); + } catch (const yexception& e) { + return Result<TDumpResult>(EStatus::INTERNAL_ERROR, e.what()); + } +} + } // NDump } // NYdb diff --git a/ydb/public/lib/ydb_cli/dump/dump_impl.h b/ydb/public/lib/ydb_cli/dump/dump_impl.h index 0ea0b835bd..287e5a48bd 100644 --- a/ydb/public/lib/ydb_cli/dump/dump_impl.h +++ b/ydb/public/lib/ydb_cli/dump/dump_impl.h @@ -11,6 +11,10 @@ public: TDumpResult Dump(const TString& dbPath, const TString& fsPath, const TDumpSettings& settings = {}); + TDumpResult DumpCluster(const TString& fsPath); + + TDumpResult DumpDatabase(const TString& database, const TString& fsPath); + private: const TDriver& Driver; std::shared_ptr<TLog> Log; diff --git a/ydb/public/lib/ydb_cli/dump/files/files.cpp b/ydb/public/lib/ydb_cli/dump/files/files.cpp index 90b6074416..496909c623 100644 --- a/ydb/public/lib/ydb_cli/dump/files/files.cpp +++ b/ydb/public/lib/ydb_cli/dump/files/files.cpp @@ -14,6 +14,10 @@ enum EFilesType { INCOMPLETE, EMPTY, CREATE_VIEW, + DATABASE, + CREATE_USER, + CREATE_GROUP, + ALTER_GROUP, }; static constexpr TFileInfo FILES_INFO[] = { @@ -28,6 +32,10 @@ static constexpr TFileInfo FILES_INFO[] = { {"incomplete", "incomplete"}, {"empty_dir", "empty_dir"}, {"create_view.sql", "view"}, + {"database.pb", "database description"}, + {"create_user.sql", "users"}, + {"create_group.sql", "groups"}, + {"alter_group.sql", "group members"}, }; const TFileInfo& TableScheme() { @@ -74,4 +82,20 @@ const TFileInfo& CreateView() { return FILES_INFO[CREATE_VIEW]; } +const TFileInfo& Database() { + return FILES_INFO[DATABASE]; +} + +const TFileInfo& CreateUser() { + return FILES_INFO[CREATE_USER]; +} + +const TFileInfo& CreateGroup() { + return FILES_INFO[CREATE_GROUP]; +} + +const TFileInfo& AlterGroup() { + return FILES_INFO[ALTER_GROUP]; +} + } // NYdb::NDump::NFiles diff --git a/ydb/public/lib/ydb_cli/dump/files/files.h b/ydb/public/lib/ydb_cli/dump/files/files.h index 3e65618724..6b0fdca158 100644 --- a/ydb/public/lib/ydb_cli/dump/files/files.h +++ b/ydb/public/lib/ydb_cli/dump/files/files.h @@ -18,5 +18,9 @@ const TFileInfo& IncompleteData(); const TFileInfo& Incomplete(); const TFileInfo& Empty(); const TFileInfo& CreateView(); +const TFileInfo& Database(); +const TFileInfo& CreateUser(); +const TFileInfo& CreateGroup(); +const TFileInfo& AlterGroup(); } // NYdb::NDump:NFiles diff --git a/ydb/public/lib/ydb_cli/dump/util/util.cpp b/ydb/public/lib/ydb_cli/dump/util/util.cpp index 48e8be3b20..2e35ac36ca 100644 --- a/ydb/public/lib/ydb_cli/dump/util/util.cpp +++ b/ydb/public/lib/ydb_cli/dump/util/util.cpp @@ -6,6 +6,7 @@ namespace NYdb::NDump { using namespace NScheme; using namespace NTable; +using namespace NCms; TStatus DescribeTable(TTableClient& tableClient, const TString& path, TMaybe<TTableDescription>& out) { auto func = [&path, &out](TSession session) { @@ -40,4 +41,22 @@ TStatus ModifyPermissions(TSchemeClient& schemeClient, const TString& path, cons }); } +TListDirectoryResult ListDirectory(TSchemeClient& schemeClient, const TString& path, const TListDirectorySettings& settings) { + return NConsoleClient::RetryFunction([&]() -> TListDirectoryResult { + return schemeClient.ListDirectory(path, settings).ExtractValueSync(); + }); +} + +TListDatabasesResult ListDatabases(TCmsClient& cmsClient, const TListDatabasesSettings& settings) { + return NConsoleClient::RetryFunction([&]() -> TListDatabasesResult { + return cmsClient.ListDatabases(settings).ExtractValueSync(); + }); +} + +TGetDatabaseStatusResult GetDatabaseStatus(TCmsClient& cmsClient, const std::string& path, const TGetDatabaseStatusSettings& settings) { + return NConsoleClient::RetryFunction([&]() -> TGetDatabaseStatusResult { + return cmsClient.GetDatabaseStatus(path, settings).ExtractValueSync(); + }); +} + } diff --git a/ydb/public/lib/ydb_cli/dump/util/util.h b/ydb/public/lib/ydb_cli/dump/util/util.h index 8d9fef2310..25c6da5a17 100644 --- a/ydb/public/lib/ydb_cli/dump/util/util.h +++ b/ydb/public/lib/ydb_cli/dump/util/util.h @@ -1,8 +1,9 @@ #pragma once -#include <ydb-cpp-sdk/client/types/status/status.h> +#include <ydb-cpp-sdk/client/cms/cms.h> #include <ydb-cpp-sdk/client/scheme/scheme.h> #include <ydb-cpp-sdk/client/table/table.h> +#include <ydb-cpp-sdk/client/types/status/status.h> #include <util/generic/maybe.h> #include <util/string/builder.h> @@ -55,4 +56,19 @@ TStatus ModifyPermissions( NScheme::TSchemeClient& schemeClient, const TString& path, const NScheme::TModifyPermissionsSettings& settings = {}); -} + +NScheme::TListDirectoryResult ListDirectory( + NScheme::TSchemeClient& schemeClient, + const TString& path, + const NScheme::TListDirectorySettings& settings = {}); + +NCms::TListDatabasesResult ListDatabases( + NCms::TCmsClient& cmsClient, + const NCms::TListDatabasesSettings& settings = {}); + +NCms::TGetDatabaseStatusResult GetDatabaseStatus( + NCms::TCmsClient& cmsClient, + const std::string& path, + const NCms::TGetDatabaseStatusSettings& settings = {}); + +} // namespace NYDB::NDump diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h index 02e5f11257..15adcfac40 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/cms/cms.h @@ -3,6 +3,7 @@ #include <ydb-cpp-sdk/client/driver/driver.h> namespace Ydb::Cms { + class CreateDatabaseRequest; class ListDatabasesResult; class GetDatabaseStatusResult; @@ -34,9 +35,7 @@ private: using TAsyncListDatabasesResult = NThreading::TFuture<TListDatabasesResult>; -struct TGetDatabaseStatusSettings : public TOperationRequestSettings<TGetDatabaseStatusSettings> { - FLUENT_SETTING(std::string, Path); -}; +struct TGetDatabaseStatusSettings : public TOperationRequestSettings<TGetDatabaseStatusSettings> {}; enum class EState { StateUnspecified = 0, @@ -165,6 +164,9 @@ public: const TDatabaseQuotas& GetDatabaseQuotas() const; const TScaleRecommenderPolicies& GetScaleRecommenderPolicies() const; + // Fills CreateDatabaseRequest proto from this database status + void SerializeTo(Ydb::Cms::CreateDatabaseRequest& request) const; + private: std::string Path_; EState State_; @@ -184,8 +186,9 @@ public: explicit TCmsClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); TAsyncListDatabasesResult ListDatabases(const TListDatabasesSettings& settings = TListDatabasesSettings()); - TAsyncGetDatabaseStatusResult GetDatabaseStatus(const TGetDatabaseStatusSettings& settings = TGetDatabaseStatusSettings()); - + TAsyncGetDatabaseStatusResult GetDatabaseStatus(const std::string& path, + const TGetDatabaseStatusSettings& settings = TGetDatabaseStatusSettings()); + private: class TImpl; std::shared_ptr<TImpl> Impl_; diff --git a/ydb/public/sdk/cpp/src/client/cms/cms.cpp b/ydb/public/sdk/cpp/src/client/cms/cms.cpp index da178521d9..3951f95696 100644 --- a/ydb/public/sdk/cpp/src/client/cms/cms.cpp +++ b/ydb/public/sdk/cpp/src/client/cms/cms.cpp @@ -174,6 +174,71 @@ const TScaleRecommenderPolicies& TGetDatabaseStatusResult::GetScaleRecommenderPo return ScaleRecommenderPolicies_; } +void TGetDatabaseStatusResult::SerializeTo(Ydb::Cms::CreateDatabaseRequest& request) const { + request.set_path(Path_); + if (std::holds_alternative<NCms::TResources>(ResourcesKind_)) { + const auto& resources = std::get<NCms::TResources>(ResourcesKind_); + for (const auto& storageUnit : resources.StorageUnits) { + auto* protoUnit = request.mutable_resources()->add_storage_units(); + protoUnit->set_unit_kind(storageUnit.UnitKind); + protoUnit->set_count(storageUnit.Count); + } + for (const auto& computationalUnit : resources.ComputationalUnits) { + auto* protoUnit = request.mutable_resources()->add_computational_units(); + protoUnit->set_unit_kind(computationalUnit.UnitKind); + protoUnit->set_count(computationalUnit.Count); + protoUnit->set_availability_zone(computationalUnit.AvailabilityZone); + } + } else if (std::holds_alternative<NCms::TSharedResources>(ResourcesKind_)) { + const auto& resources = std::get<NCms::TSharedResources>(ResourcesKind_); + for (const auto& storageUnit : resources.StorageUnits) { + auto* protoUnit = request.mutable_shared_resources()->add_storage_units(); + protoUnit->set_unit_kind(storageUnit.UnitKind); + protoUnit->set_count(storageUnit.Count); + } + for (const auto& computationalUnit : resources.ComputationalUnits) { + auto* protoUnit = request.mutable_shared_resources()->add_computational_units(); + protoUnit->set_unit_kind(computationalUnit.UnitKind); + protoUnit->set_count(computationalUnit.Count); + protoUnit->set_availability_zone(computationalUnit.AvailabilityZone); + } + } else if (std::holds_alternative<NCms::TServerlessResources>(ResourcesKind_)) { + const auto& resources = std::get<NCms::TServerlessResources>(ResourcesKind_); + request.mutable_serverless_resources()->set_shared_database_path(resources.SharedDatabasePath); + } + + for (const auto& quota : SchemaOperationQuotas_.LeakyBucketQuotas) { + auto protoQuota = request.mutable_schema_operation_quotas()->add_leaky_bucket_quotas(); + protoQuota->set_bucket_seconds(quota.BucketSeconds); + protoQuota->set_bucket_size(quota.BucketSize); + } + + request.mutable_database_quotas()->set_data_size_hard_quota(DatabaseQuotas_.DataSizeHardQuota); + request.mutable_database_quotas()->set_data_size_soft_quota(DatabaseQuotas_.DataSizeSoftQuota); + request.mutable_database_quotas()->set_data_stream_shards_quota(DatabaseQuotas_.DataStreamShardsQuota); + request.mutable_database_quotas()->set_data_stream_reserved_storage_quota(DatabaseQuotas_.DataStreamReservedStorageQuota); + request.mutable_database_quotas()->set_ttl_min_run_internal_seconds(DatabaseQuotas_.TtlMinRunInternalSeconds); + + for (const auto& quota : DatabaseQuotas_.StorageQuotas) { + auto protoQuota = request.mutable_database_quotas()->add_storage_quotas(); + protoQuota->set_unit_kind(quota.UnitKind); + protoQuota->set_data_size_hard_quota(quota.DataSizeHardQuota); + protoQuota->set_data_size_soft_quota(quota.DataSizeSoftQuota); + } + + for (const auto& policy : ScaleRecommenderPolicies_.Policies) { + auto* protoPolicy = request.mutable_scale_recommender_policies()->add_policies(); + if (std::holds_alternative<NCms::TTargetTrackingPolicy>(policy.Policy)) { + const auto& targetTracking = std::get<NCms::TTargetTrackingPolicy>(policy.Policy); + auto* protoTargetTracking = protoPolicy->mutable_target_tracking_policy(); + if (std::holds_alternative<NCms::TTargetTrackingPolicy::TAverageCpuUtilizationPercent>(targetTracking.Target)) { + const auto& target = std::get<NCms::TTargetTrackingPolicy::TAverageCpuUtilizationPercent>(targetTracking.Target); + protoTargetTracking->set_average_cpu_utilization_percent(target); + } + } + } +} + class TCmsClient::TImpl : public TClientImplCommon<TCmsClient::TImpl> { public: TImpl(std::shared_ptr<TGRpcConnectionsImpl>&& connections, const TCommonClientSettings& settings) @@ -206,9 +271,9 @@ public: return promise.GetFuture(); } - TAsyncGetDatabaseStatusResult GetDatabaseStatus(const TGetDatabaseStatusSettings& settings) { + TAsyncGetDatabaseStatusResult GetDatabaseStatus(const std::string& path, const TGetDatabaseStatusSettings& settings) { Ydb::Cms::GetDatabaseStatusRequest request; - request.set_path(settings.Path_); + request.set_path(path); auto promise = NThreading::NewPromise<TGetDatabaseStatusResult>(); @@ -242,8 +307,11 @@ TAsyncListDatabasesResult TCmsClient::ListDatabases(const TListDatabasesSettings return Impl_->ListDatabases(settings); } -TAsyncGetDatabaseStatusResult TCmsClient::GetDatabaseStatus(const TGetDatabaseStatusSettings& settings) { - return Impl_->GetDatabaseStatus(settings); +TAsyncGetDatabaseStatusResult TCmsClient::GetDatabaseStatus( + const std::string& path, + const TGetDatabaseStatusSettings& settings) +{ + return Impl_->GetDatabaseStatus(path, settings); } } // namespace NYdb::NCms diff --git a/ydb/tests/functional/ydb_cli/test_ydb_backup.py b/ydb/tests/functional/ydb_cli/test_ydb_backup.py index bf5ee44df4..301b9bd1be 100644 --- a/ydb/tests/functional/ydb_cli/test_ydb_backup.py +++ b/ydb/tests/functional/ydb_cli/test_ydb_backup.py @@ -4,9 +4,10 @@ from ydb.tests.library.harness.kikimr_runner import KiKiMR from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.oss.ydb_sdk_import import ydb -from hamcrest import assert_that, is_, is_not, contains_inanyorder, has_items -import os +from hamcrest import assert_that, is_, is_not, contains_inanyorder, has_items, equal_to +import enum import logging +import os import pytest import yatest @@ -186,18 +187,25 @@ def is_permissions_the_same(scheme_client, path_left, path_right): return True -def list_all_dirs(prefix, path=""): +@enum.unique +class ListMode(enum.IntEnum): + DIRS = 0, + FILES = 1, + + +def fs_recursive_list(prefix, mode=ListMode.DIRS, path=""): paths = [] full_path = os.path.join(prefix, path) logger.debug("prefix# " + prefix + " path# " + path) for item in os.listdir(full_path): item_path = os.path.join(full_path, item) if os.path.isdir(item_path): - paths.append(os.path.join(path, item)) - paths += list_all_dirs(prefix, os.path.join(path, item)) - else: - # don't list regular files - pass + if mode == ListMode.DIRS: + paths.append(os.path.join(path, item)) + paths += fs_recursive_list(prefix, mode, os.path.join(path, item)) + elif os.path.isfile(item_path): + if mode == ListMode.FILES: + paths.append(os.path.join(path, item)) return paths @@ -244,12 +252,12 @@ class BaseTestBackupInFiles(object): ) logger.debug("std_out:\n" + execution.std_out.decode('utf-8')) - list_all_dirs(backup_files_dir) - logger.debug("list_all_dirs(backup_files_dir)# " + str(list_all_dirs(backup_files_dir))) + fs_recursive_list(backup_files_dir) + logger.debug("fs_recursive_list(backup_files_dir)# " + str(fs_recursive_list(backup_files_dir))) logger.debug("expected_dirs# " + str(expected_dirs)) assert_that( - list_all_dirs(backup_files_dir), + fs_recursive_list(backup_files_dir), has_items(*expected_dirs) ) @@ -1286,3 +1294,133 @@ class TestRestoreNoData(BaseTestBackupInFiles): is_data_the_same(session, "/Root/folder/table", "/Root/restored/table"), is_(False) ) + + +class BaseTestClusterBackupInFiles(object): + @classmethod + def setup_class(cls): + cls.cluster = KiKiMR(KikimrConfigGenerator(extra_feature_flags=["enable_resource_pools"])) + cls.cluster.start() + + cls.root_dir = "/Root" + cls.database = os.path.join(cls.root_dir, "db1") + + cls.cluster.create_database( + cls.database, + storage_pool_units_count={ + 'hdd': 1 + }, + timeout_seconds=100 + ) + + cls.database_nodes = cls.cluster.register_and_start_slots(cls.database, count=3) + cls.cluster.wait_tenant_up(cls.database) + + driver_config = ydb.DriverConfig( + database=cls.database, + endpoint="%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].port)) + cls.driver = ydb.Driver(driver_config) + cls.driver.wait(timeout=4) + + @classmethod + def teardown_class(cls): + cls.cluster.unregister_and_stop_slots(cls.database_nodes) + cls.cluster.stop() + + @pytest.fixture(autouse=True, scope='class') + @classmethod + def set_test_name(cls, request): + cls.test_name = request.node.name + + @classmethod + def create_backup(cls, command, expected_files, additional_args=[]): + backup_files_dir = output_path(cls.test_name, "backup_files_dir") + execution = yatest.common.execute( + [ + backup_bin(), + "--verbose", + "--assume-yes", + "--endpoint", "grpc://localhost:%d" % cls.cluster.nodes[1].grpc_port, + ] + + command + + ["--output", backup_files_dir] + + additional_args + ) + + list_result = fs_recursive_list(backup_files_dir, ListMode.FILES) + + logger.debug("std_out:\n" + execution.std_out.decode('utf-8')) + logger.debug("fs_recursive_list(backup_files_dir)# " + str(list_result)) + logger.debug("expected_files# " + str(expected_files)) + + assert_that( + len(list_result), + equal_to(len(expected_files)) + ) + + assert_that( + list_result, + has_items(*expected_files) + ) + + @classmethod + def create_cluster_backup(cls, expected_files, additional_args=[]): + cls.create_backup( + [ + "admin", "cluster", "dump", + ], + expected_files, + additional_args + ) + + @classmethod + def create_database_backup(cls, expected_files, additional_args=[]): + cls.create_backup( + [ + "--database", cls.database, + "admin", "database", "dump", + ], + expected_files, + additional_args + ) + + +class TestClusterBackup(BaseTestClusterBackupInFiles): + def test_cluster_backup(self): + session = self.driver.table_client.session().create() + create_table_with_data(session, "db1/table") + + self.create_cluster_backup(expected_files=[ + # cluster metadata + "permissions.pb", + "create_user.sql", + "create_group.sql", + "alter_group.sql", + + # database metadata + "Root/db1/database.pb", + "Root/db1/permissions.pb", + "Root/db1/create_user.sql", + "Root/db1/create_group.sql", + "Root/db1/alter_group.sql", + ]) + + +class TestDatabaseBackup(BaseTestClusterBackupInFiles): + def test_database_backup(self): + session = self.driver.table_client.session().create() + create_table_with_data(session, "db1/table") + + self.create_database_backup(expected_files=[ + # database metadata + "database.pb", + "permissions.pb", + "create_user.sql", + "create_group.sql", + "alter_group.sql", + + # database table + "table/scheme.pb", + "table/permissions.pb", + "table/data_00.csv", + ]) |