Introduce delayed restore manager and use it for views and external tables (#14597)

Co-authored-by: Ilnaz Nizametdinov <i.nizametdinov@gmail.com>

3 files changed, 221 insertions, 84 deletions

diff --git a/ydb/public/lib/ydb_cli/dump/restore_impl.cpp b/ydb/public/lib/ydb_cli/dump/restore_impl.cpp
index 41a0a2577f..58c1bb0c54 100644
--- a/ydb/public/lib/ydb_cli/dump/restore_impl.cpp
+++ b/ydb/public/lib/ydb_cli/dump/restore_impl.cpp
@@ -301,6 +301,112 @@ TString TBatch::GetLocation() const {
     return result;
 }
 
+TDelayedRestoreCall::TDelayedRestoreCall(
+    ESchemeEntryType type,
+    TFsPath fsPath,
+    TString dbPath,
+    TRestoreSettings settings,
+    bool isAlreadyExisting
+)
+    : Type(type)
+    , FsPath(fsPath)
+    , DbPath(dbPath)
+    , Settings(settings)
+    , IsAlreadyExisting(isAlreadyExisting)
+{}
+
+TDelayedRestoreCall::TDelayedRestoreCall(
+    ESchemeEntryType type,
+    TFsPath fsPath,
+    TString dbRestoreRoot,
+    TString dbPathRelativeToRestoreRoot,
+    TRestoreSettings settings,
+    bool isAlreadyExisting
+)
+    : Type(type)
+    , FsPath(fsPath)
+    , DbPath(TTwoComponentPath(dbRestoreRoot, dbPathRelativeToRestoreRoot))
+    , Settings(settings)
+    , IsAlreadyExisting(isAlreadyExisting)
+{}
+
+int TDelayedRestoreCall::GetOrder() const {
+    switch (Type) {
+        case ESchemeEntryType::View:
+            return std::numeric_limits<int>::max();
+        default:
+            return 0;
+    }
+}
+
+auto operator<=>(const TDelayedRestoreCall& lhs, const TDelayedRestoreCall& rhs) {
+    return lhs.GetOrder() <=> rhs.GetOrder();
+}
+
+TRestoreResult TDelayedRestoreManager::Restore(const TDelayedRestoreCall& call) {
+    switch (call.Type) {
+        case ESchemeEntryType::View: {
+            const auto& [dbRestoreRoot, dbPathRelativeToRestoreRoot] = std::get<TDelayedRestoreCall::TTwoComponentPath>(call.DbPath);
+            return Client->RestoreView(call.FsPath, dbRestoreRoot, dbPathRelativeToRestoreRoot, call.Settings, call.IsAlreadyExisting);
+        }
+        case ESchemeEntryType::ExternalTable: {
+            const auto& dbPath = std::get<TDelayedRestoreCall::TSimplePath>(call.DbPath);
+            return Client->RestoreExternalTable(call.FsPath, dbPath, call.Settings, call.IsAlreadyExisting);
+        }
+        default:
+            ythrow TBadArgumentException() << "Attempting to restore an unexpected object from: " << call.FsPath;
+    }
+}
+
+bool TDelayedRestoreManager::ShouldRetry(const TRestoreResult& result, ESchemeEntryType type) {
+    switch (type) {
+        case ESchemeEntryType::View:
+            return result.GetStatus() == EStatus::SCHEME_ERROR;
+        default:
+            return false;
+    }
+}
+
+TRestoreResult TDelayedRestoreManager::RestoreWithRetries(TVector<TDelayedRestoreCall>&& callsToRetry) {
+    bool stopRetries = false;
+    TVector<TDelayedRestoreCall> nextRound;
+    while (!callsToRetry.empty()) {
+        nextRound.clear();
+        for (const auto& call : callsToRetry) {
+            auto result = Restore(call);
+            if (!result.IsSuccess()) {
+                if (stopRetries || !ShouldRetry(result, call.Type)) {
+                    return result;
+                }
+                nextRound.emplace_back(call);
+            }
+        }
+        // errors are persistent
+        stopRetries = nextRound.size() == callsToRetry.size();
+        std::swap(nextRound, callsToRetry);
+    }
+    return Result<TRestoreResult>();
+}
+
+void TDelayedRestoreManager::SetClient(TRestoreClient& client) {
+    Client = &client;
+}
+
+TRestoreResult TDelayedRestoreManager::RestoreDelayed() {
+    std::sort(Calls.begin(), Calls.end());
+    TVector<TDelayedRestoreCall> callsToRetry;
+    for (const auto& call : Calls) {
+        auto result = Restore(call);
+        if (!result.IsSuccess()) {
+            if (!ShouldRetry(result, call.Type)) {
+                return result;
+            }
+            callsToRetry.emplace_back(call);
+        }
+    }
+    return RestoreWithRetries(std::move(callsToRetry));
+}
+
 } // NPrivate
 
 TRestoreClient::TRestoreClient(const TDriver& driver, const std::shared_ptr<TLog>& log)
@@ -316,46 +422,7 @@ TRestoreClient::TRestoreClient(const TDriver& driver, const std::shared_ptr<TLog
     , Log(log)
     , DriverConfig(driver.GetConfig())
 {
-}
-
-TRestoreResult TRestoreClient::RetryViewRestoration() {
-    auto result = Result<TRestoreResult>();
-
-    if (!ViewRestorationCalls.empty()) {
-        TVector<TRestoreViewCall> calls;
-        TMaybe<TRestoreResult> lastFail;
-        size_t size;
-        do {
-            calls.clear();
-            lastFail.Clear();
-            size = ViewRestorationCalls.size();
-            std::swap(calls, ViewRestorationCalls);
-
-            for (const auto& [fsPath, dbRestoreRoot, dbPathRelativeToRestoreRoot, settings, isAlreadyExisting] : calls) {
-                auto result = RestoreView(fsPath, dbRestoreRoot, dbPathRelativeToRestoreRoot, settings, isAlreadyExisting);
-                if (!result.IsSuccess()) {
-                    lastFail = std::move(result);
-                }
-            }
-        } while (!ViewRestorationCalls.empty() && ViewRestorationCalls.size() < size);
-
-        // retries could not fix the errors
-        if (!ViewRestorationCalls.empty() || lastFail) {
-            result = *lastFail;
-        }
-    }
-
-    return result;
-}
-
-TRestoreResult TRestoreClient::RestoreExternalTables() {
-    for (const auto& [fsPath, dbPath, settings, isAlreadyExisting] : ExternalTableRestorationCalls) {
-        auto result = RestoreExternalTable(fsPath, dbPath, settings, isAlreadyExisting);
-        if (!result.IsSuccess()) {
-            return result;
-        }
-    }
-    return Result<TRestoreResult>();
+    DelayedRestoreManager.SetClient(*this);
 }
 
 TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbPath, const TRestoreSettings& settings) {
@@ -394,10 +461,7 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP
 
     // restore
     auto restoreResult = RestoreFolder(fsPath, dbPath, "", settings, oldEntries);
-    if (auto result = RetryViewRestoration(); !result.IsSuccess()) {
-        restoreResult = result;
-    }
-    if (auto result = RestoreExternalTables(); !result.IsSuccess()) {
+    if (auto result = DelayedRestoreManager.RestoreDelayed(); !result.IsSuccess()) {
         restoreResult = result;
     }
 
@@ -775,10 +839,7 @@ TRestoreResult TRestoreClient::RestoreDatabaseImpl(const TString& fsPath, const
 
     if (settings.WithContent_) {
         auto restoreResult = RestoreFolder(fsPath, dbPath, "", {}, {});
-        if (auto result = RetryViewRestoration(); !result.IsSuccess()) {
-            restoreResult = result;
-        }
-        if (auto result = RestoreExternalTables(); !result.IsSuccess()) {
+        if (auto result = DelayedRestoreManager.RestoreDelayed(); !result.IsSuccess()) {
             restoreResult = result;
         }
         return restoreResult;
@@ -905,8 +966,7 @@ TRestoreResult TRestoreClient::RestoreFolder(
     }
 
     if (IsFileExists(fsPath.Child(NFiles::CreateView().FileName))) {
-        // delay view restoration
-        ViewRestorationCalls.emplace_back(fsPath, dbRestoreRoot, dbPathRelativeToRestoreRoot, settings, oldEntries.contains(objectDbPath));
+        DelayedRestoreManager.Add(ESchemeEntryType::View, fsPath, dbRestoreRoot, dbPathRelativeToRestoreRoot, settings, oldEntries.contains(objectDbPath));
         return Result<TRestoreResult>();
     }
 
@@ -927,8 +987,7 @@ TRestoreResult TRestoreClient::RestoreFolder(
     }
 
     if (IsFileExists(fsPath.Child(NFiles::CreateExternalTable().FileName))) {
-        // delay external table restoration
-        ExternalTableRestorationCalls.emplace_back(fsPath, objectDbPath, settings, oldEntries.contains(objectDbPath));
+        DelayedRestoreManager.Add(ESchemeEntryType::ExternalTable, fsPath, objectDbPath, settings, oldEntries.contains(objectDbPath));
         return Result<TRestoreResult>();
     }
 
@@ -947,8 +1006,7 @@ TRestoreResult TRestoreClient::RestoreFolder(
         } else if (IsFileExists(child.Child(NFiles::Empty().FileName))) {
             result = RestoreEmptyDir(child, childDbPath, settings, oldEntries.contains(childDbPath));
         } else if (IsFileExists(child.Child(NFiles::CreateView().FileName))) {
-            // delay view restoration
-            ViewRestorationCalls.emplace_back(child, dbRestoreRoot, Join('/', dbPathRelativeToRestoreRoot, child.GetName()), settings, oldEntries.contains(childDbPath));
+            DelayedRestoreManager.Add(ESchemeEntryType::View, child, dbRestoreRoot, Join('/', dbPathRelativeToRestoreRoot, child.GetName()), settings, oldEntries.contains(childDbPath));
         } else if (IsFileExists(child.Child(NFiles::CreateTopic().FileName))) {
             result = RestoreTopic(child, childDbPath, settings, oldEntries.contains(childDbPath));
         } else if (IsFileExists(child.Child(NFiles::CreateCoordinationNode().FileName))) {
@@ -958,8 +1016,7 @@ TRestoreResult TRestoreClient::RestoreFolder(
         } else if (IsFileExists(child.Child(NFiles::CreateExternalDataSource().FileName))) {
             result = RestoreExternalDataSource(child, childDbPath, settings, oldEntries.contains(childDbPath));
         } else if (IsFileExists(child.Child(NFiles::CreateExternalTable().FileName))) {
-            // delay external table restoration
-            ExternalTableRestorationCalls.emplace_back(child, childDbPath, settings, oldEntries.contains(childDbPath));
+            DelayedRestoreManager.Add(ESchemeEntryType::ExternalTable, child, childDbPath, settings, oldEntries.contains(childDbPath));
         } else if (child.IsDirectory()) {
             result = RestoreFolder(child, dbRestoreRoot, Join('/', dbPathRelativeToRestoreRoot, child.GetName()), settings, oldEntries);
         }
@@ -971,7 +1028,7 @@ TRestoreResult TRestoreClient::RestoreFolder(
 
     const bool dbPathExists = oldEntries.contains(dbPath);
     if (!result.Defined() && !dbPathExists) {
-        // This situation occurs when all the children of the folder are views or external tables.
+        // This situation arises when all the children of the file system path are scheme objects with a delayed restoration.
         return RestoreEmptyDir(fsPath, dbPath, settings, dbPathExists);
     }
 
@@ -1017,9 +1074,6 @@ TRestoreResult TRestoreClient::RestoreView(
 
     if (result.GetStatus() == EStatus::SCHEME_ERROR) {
         LOG_I("Failed to create " << dbPath.Quote() << ". Will retry.");
-        // Scheme error happens when the view depends on a table (or a view) that is not yet restored.
-        // Instead of tracking view dependencies, we simply retry the creation of the view later.
-        ViewRestorationCalls.emplace_back(fsPath, dbRestoreRoot, dbPathRelativeToRestoreRoot, settings, isAlreadyExisting);
     } else {
         LOG_E("Failed to create " << dbPath.Quote());
     }
diff --git a/ydb/public/lib/ydb_cli/dump/restore_impl.h b/ydb/public/lib/ydb_cli/dump/restore_impl.h
index 315ba9204b..f7f9b233fd 100644
--- a/ydb/public/lib/ydb_cli/dump/restore_impl.h
+++ b/ydb/public/lib/ydb_cli/dump/restore_impl.h
@@ -22,6 +22,8 @@ namespace NYdb::NDump {
 extern const char DOC_API_TABLE_VERSION_ATTR[23];
 extern const char DOC_API_REQUEST_TYPE[22];
 
+class TRestoreClient;
+
 namespace NPrivate {
 
 class TBatch;
@@ -123,6 +125,58 @@ public:
     virtual void Wait() = 0;
 };
 
+struct TDelayedRestoreCall {
+    using TSimplePath = TString;
+
+    struct TTwoComponentPath {
+        TString RestoreRoot;
+        TString RelativeToRestoreRoot;
+    };
+
+    NScheme::ESchemeEntryType Type;
+    TFsPath FsPath;
+    std::variant<TSimplePath, TTwoComponentPath> DbPath;
+    TRestoreSettings Settings;
+    bool IsAlreadyExisting;
+
+    TDelayedRestoreCall(
+        NScheme::ESchemeEntryType type,
+        TFsPath fsPath,
+        TString dbPath,
+        TRestoreSettings settings,
+        bool isAlreadyExisting
+    );
+
+    TDelayedRestoreCall(
+        NScheme::ESchemeEntryType type,
+        TFsPath fsPath,
+        TString dbRestoreRoot,
+        TString dbPathRelativeToRestoreRoot,
+        TRestoreSettings settings,
+        bool isAlreadyExisting
+    );
+
+    int GetOrder() const;
+};
+
+class TDelayedRestoreManager {
+    TVector<TDelayedRestoreCall> Calls;
+    TRestoreClient* Client = nullptr;
+
+    TRestoreResult Restore(const TDelayedRestoreCall& call);
+    static bool ShouldRetry(const TRestoreResult& result, NScheme::ESchemeEntryType type);
+    TRestoreResult RestoreWithRetries(TVector<TDelayedRestoreCall>&& calls);
+
+public:
+    void SetClient(TRestoreClient& client);
+    TRestoreResult RestoreDelayed();
+
+    template <typename... Args>
+    void Add(NScheme::ESchemeEntryType type, Args&&... args) {
+        Calls.emplace_back(type, std::forward<Args>(args)...);
+    }
+};
+
 } // NPrivate
 
 class TRestoreClient {
@@ -148,8 +202,6 @@ class TRestoreClient {
     TRestoreResult FindClusterRootPath();
     TRestoreResult ReplaceClusterRoot(TString& outPath);
     TRestoreResult WaitForAvailableNodes(const TString& database, TDuration waitDuration);
-    TRestoreResult RetryViewRestoration();
-    TRestoreResult RestoreExternalTables();
 
     TRestoreResult RestoreClusterRoot(const TFsPath& fsPath);
     TRestoreResult RestoreDatabases(const TFsPath& fsPath, const TRestoreClusterSettings& settings);
@@ -187,29 +239,10 @@ private:
     std::shared_ptr<TLog> Log;
     // Used to creating child drivers with different database settings.
     TDriverConfig DriverConfig;
-
-    struct TRestoreViewCall {
-        TFsPath FsPath;
-        TString DbRestoreRoot;
-        TString DbPathRelativeToRestoreRoot;
-        TRestoreSettings Settings;
-        bool IsAlreadyExisting;
-    };
-    // Views usually depend on other objects.
-    // If the dependency is not created yet, then the view restoration will fail.
-    // We retry failed view creation attempts until either all views are created, or the errors are persistent.
-    TVector<TRestoreViewCall> ViewRestorationCalls;
-
-    struct TRestoreExternalTableCall {
-        TFsPath FsPath;
-        TString DbPath;
-        TRestoreSettings Settings;
-        bool IsAlreadyExisting;
-    };
-    // External Tables depend on External Data Sources and need to be restored after them.
-    TVector<TRestoreExternalTableCall> ExternalTableRestorationCalls;
-
     TString ClusterRootPath;
+    NPrivate::TDelayedRestoreManager DelayedRestoreManager;
+
+    friend class NPrivate::TDelayedRestoreManager;
 }; // TRestoreClient
 
 } // NYdb::NDump
diff --git a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp
index 78766edd76..ba54f5c725 100644
--- a/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp
+++ b/ydb/services/ydb/backup_ut/ydb_backup_ut.cpp
@@ -759,6 +759,35 @@ void TestViewReferenceTableIsPreserved(
     TestViewReferenceTableIsPreserved(view, table, view, session, std::move(backup), std::move(restore));
 }
 
+void TestViewDependentOnAnotherViewIsRestored(
+    const char* baseView, const char* dependentView, NQuery::TSession& session,
+    TBackupFunction&& backup, TRestoreFunction&& restore
+) {
+    ExecuteQuery(session, Sprintf(R"(
+                CREATE VIEW `%s` WITH security_invoker = TRUE AS SELECT 1 AS Key;
+            )", baseView
+        ), true
+    );
+    ExecuteQuery(session, Sprintf(R"(
+                CREATE VIEW `%s` WITH security_invoker = TRUE AS SELECT * FROM `%s`;
+            )", dependentView, baseView
+        ), true
+    );
+    const auto originalContent = GetTableContent(session, dependentView);
+
+    backup();
+
+    ExecuteQuery(session, Sprintf(R"(
+                DROP VIEW `%s`;
+                DROP VIEW `%s`;
+            )", baseView, dependentView
+        ), true
+    );
+
+    restore();
+    CompareResults(GetTableContent(session, dependentView), originalContent);
+}
+
 void TestTopicSettingsArePreserved(
     const char* topic, NQuery::TSession& session, NTopic::TTopicClient& topicClient,
     TBackupFunction&& backup, TRestoreFunction&& restore
@@ -1325,6 +1354,27 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
         );
     }
 
+    Y_UNIT_TEST(RestoreViewDependentOnAnotherView) {
+        TKikimrWithGrpcAndRootSchema server;
+        server.GetRuntime()->GetAppData().FeatureFlags.SetEnableViews(true);
+        auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort())));
+        NQuery::TQueryClient queryClient(driver);
+        auto session = queryClient.GetSession().ExtractValueSync().GetSession();
+        TTempDir tempDir;
+        const auto& pathToBackup = tempDir.Path();
+
+        constexpr const char* baseView = "/Root/baseView";
+        constexpr const char* dependentView = "/Root/dependentView";
+
+        TestViewDependentOnAnotherViewIsRestored(
+            baseView,
+            dependentView,
+            session,
+            CreateBackupLambda(driver, pathToBackup),
+            CreateRestoreLambda(driver, pathToBackup)
+        );
+    }
+
     Y_UNIT_TEST(RestoreKesusResources) {
         TKikimrWithGrpcAndRootSchema server;
         auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort())));