YQ-3557 tests preparation before resource pool enable (#7818)

12 files changed, 88 insertions, 36 deletions

diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
index 76b1f5d379c..fac770df0b7 100644
--- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
+++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
@@ -6496,7 +6496,13 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
     }
 
     Y_UNIT_TEST(DisableResourcePools) {
-        TKikimrRunner kikimr(TKikimrSettings().SetEnableResourcePools(false));
+        NKikimrConfig::TAppConfig config;
+        config.MutableFeatureFlags()->SetEnableResourcePools(false);
+
+        TKikimrRunner kikimr(NKqp::TKikimrSettings()
+            .SetAppConfig(config)
+            .SetEnableResourcePools(false));
+
         auto db = kikimr.GetTableClient();
         auto session = db.CreateSession().GetValueSync().GetSession();
 
diff --git a/ydb/core/kqp/workload_service/actors/scheme_actors.cpp b/ydb/core/kqp/workload_service/actors/scheme_actors.cpp
index 86cc634aa3a..17ca8a48fed 100644
--- a/ydb/core/kqp/workload_service/actors/scheme_actors.cpp
+++ b/ydb/core/kqp/workload_service/actors/scheme_actors.cpp
@@ -180,7 +180,7 @@ protected:
         LOG_D("Start pool fetching");
         auto event = NTableCreator::BuildSchemeCacheNavigateRequest(
             {{".metadata/workload_manager/pools", PoolId}},
-            Database,
+            Database ? Database : AppData()->TenantName,
             UserToken
         );
         event->ResultSet[0].Access |= NACLib::SelectRow;
@@ -326,7 +326,7 @@ protected:
         auto event = std::make_unique<TEvTxUserProxy::TEvProposeTransaction>();
 
         auto& schemeTx = *event->Record.MutableTransaction()->MutableModifyScheme();
-        schemeTx.SetWorkingDir(JoinPath({Database, ".metadata/workload_manager/pools"}));
+        schemeTx.SetWorkingDir(JoinPath({Database ? Database : AppData()->TenantName, ".metadata/workload_manager/pools"}));
         schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateResourcePool);
         schemeTx.SetInternal(true);
 
@@ -483,7 +483,13 @@ public:
                 }
                 return;
             case EStatus::Ok:
-                Serverless = result.DomainInfo && result.DomainInfo->IsServerless();
+                if (result.DomainInfo) {
+                    Serverless = result.DomainInfo->IsServerless();
+                    if (result.Self->Info.GetPathId() != result.DomainInfo->DomainKey.LocalPathId) {
+                        Reply(Ydb::StatusIds::UNSUPPORTED, TStringBuilder() << "Invalid database " << Database << ", domain path id is different");
+                        return;
+                    }
+                }
                 Reply(Ydb::StatusIds::SUCCESS);
                 return;
         }
@@ -500,7 +506,11 @@ public:
 protected:
     void StartRequest() override {
         LOG_D("Start database fetching");
-        auto event = NTableCreator::BuildSchemeCacheNavigateRequest({{}}, Database, UserToken);
+        auto event = NTableCreator::BuildSchemeCacheNavigateRequest(
+            {{}},
+            Database ? Database : AppData()->TenantName,
+            UserToken
+        );
         event->ResultSet[0].Operation = NSchemeCache::TSchemeCacheNavigate::OpPath;
         event->ResultSet[0].Access |= CheckAccess;
         Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(event.Release()), IEventHandle::FlagTrackDelivery);
diff --git a/ydb/core/kqp/workload_service/kqp_workload_service_impl.h b/ydb/core/kqp/workload_service/kqp_workload_service_impl.h
index 977c7ef3034..ca0b84cdd97 100644
--- a/ydb/core/kqp/workload_service/kqp_workload_service_impl.h
+++ b/ydb/core/kqp/workload_service/kqp_workload_service_impl.h
@@ -22,6 +22,7 @@ struct TDatabaseState {
     std::unordered_map<TString, std::unordered_set<TActorId>> PendingSubscriptions = {};
     bool HasDefaultPool = false;
     bool Serverless = false;
+    bool DatabaseUnsupported = false;
 
     TInstant LastUpdateTime = TInstant::Zero();
 
@@ -41,8 +42,10 @@ struct TDatabaseState {
 
         if (!EnabledResourcePoolsOnServerless && (TInstant::Now() - LastUpdateTime) > IDLE_DURATION) {
             ActorContext.Register(CreateDatabaseFetcherActor(ActorContext.SelfID, database));
-        } else {
+        } else if (!DatabaseUnsupported) {
             StartPendingRequests();
+        } else {
+            ReplyContinueError(Ydb::StatusIds::UNSUPPORTED, {NYql::TIssue(TStringBuilder() << "Unsupported database: " << database)});
         }
     }
 
@@ -65,6 +68,7 @@ struct TDatabaseState {
     }
 
     void UpdateDatabaseInfo(const TEvPrivate::TEvFetchDatabaseResponse::TPtr& ev) {
+        DatabaseUnsupported = ev->Get()->Status == Ydb::StatusIds::UNSUPPORTED;
         if (ev->Get()->Status != Ydb::StatusIds::SUCCESS) {
             ReplyContinueError(ev->Get()->Status, GroupIssues(ev->Get()->Issues, "Failed to fetch database info"));
             return;
diff --git a/ydb/core/memory_controller/memory_controller_ut.cpp b/ydb/core/memory_controller/memory_controller_ut.cpp
index 7b3b5b38070..00fab4b7e8e 100644
--- a/ydb/core/memory_controller/memory_controller_ut.cpp
+++ b/ydb/core/memory_controller/memory_controller_ut.cpp
@@ -262,7 +262,6 @@ Y_UNIT_TEST(SharedCache) {
 
     server->PrintCounters();
     UNIT_ASSERT_VALUES_EQUAL(server->SharedPageCacheCounters->ConfigLimitBytes->Val(), 32_MB);
-    UNIT_ASSERT_VALUES_EQUAL(server->SharedPageCacheCounters->MemLimitBytes->Val(), 0); // not applied yet
     UNIT_ASSERT_VALUES_EQUAL(server->SharedPageCacheCounters->ActiveLimitBytes->Val(), server->SharedPageCacheCounters->ConfigLimitBytes->Val());
 
     runtime.SimulateSleep(TDuration::Seconds(2));
diff --git a/ydb/core/sys_view/pg_tables/pg_tables.cpp b/ydb/core/sys_view/pg_tables/pg_tables.cpp
index 20d47306ed2..1965f700eb9 100644
--- a/ydb/core/sys_view/pg_tables/pg_tables.cpp
+++ b/ydb/core/sys_view/pg_tables/pg_tables.cpp
@@ -125,8 +125,13 @@ void TPgTablesScanBase::Handle(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeRe
     ExpandBatchWithStaticTables(batch);
 
     for (size_t i = 0; i < record.GetPathDescription().ChildrenSize(); ++i) {
+        const auto& childrenDescription = record.GetPathDescription().GetChildren(i);
+        if (childrenDescription.GetPathType() == NKikimrSchemeOp::EPathTypeDir) {
+            continue;
+        }
+
         TVector<TString> cellData;
-        const auto& tableName = record.GetPathDescription().GetChildren(i).GetName();
+        const auto& tableName = childrenDescription.GetName();
         const auto& tableOwner = record.GetPathDescription().GetSelf().GetOwner();
         TVector<TCell> cells = MakePgTablesRow(tableName, tableOwner, cellData);
         if (!ConvertError_.Empty()) {
diff --git a/ydb/core/sys_view/ut_common.cpp b/ydb/core/sys_view/ut_common.cpp
index 3583b14700d..10fd9f068c3 100644
--- a/ydb/core/sys_view/ut_common.cpp
+++ b/ydb/core/sys_view/ut_common.cpp
@@ -43,11 +43,16 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, ui32
     // in some tests we check data size, which depends on compaction,
     NKikimrConfig::TFeatureFlags featureFlags;
     featureFlags.SetEnableBackgroundCompaction(false);
+    featureFlags.SetEnableResourcePools(true);
     Settings->SetFeatureFlags(featureFlags);
 
     Settings->SetEnablePersistentQueryStats(enableSVP);
     Settings->SetEnableDbCounters(enableSVP);
 
+    NKikimrConfig::TAppConfig appConfig;
+    *appConfig.MutableFeatureFlags() = Settings->FeatureFlags;
+    Settings->SetAppConfig(appConfig);
+
     for (ui32 i : xrange(storagePools)) {
         TString poolName = Sprintf("test%d", i);
         Settings->AddStoragePool(poolName, TString("/Root:") + poolName, 2);
diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp
index cfbd3cbb213..cf8f7f95974 100644
--- a/ydb/core/sys_view/ut_kqp.cpp
+++ b/ydb/core/sys_view/ut_kqp.cpp
@@ -256,7 +256,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
             UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
 
             NKqp::CompareYson(R"([
-                [[5u];[0u];["/Root/Tenant1/Table1"]]
+                [[9u];[0u];["/Root/Tenant1/Table1"]]
             ])", NKqp::StreamResultToYson(it));
         }
         {
@@ -267,7 +267,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
             UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
 
             NKqp::CompareYson(R"([
-                [[6u];[0u];["/Root/Tenant2/Table2"]]
+                [[10u];[0u];["/Root/Tenant2/Table2"]]
             ])", NKqp::StreamResultToYson(it));
         }
     }
@@ -297,7 +297,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
 
             UNIT_ASSERT(result.IsSuccess());
             NKqp::CompareYson(R"([
-                [[5u];[0u];["/Root/Tenant1/Table1"]]
+                [[9u];[0u];["/Root/Tenant1/Table1"]]
             ])", FormatResultSetYson(result.GetResultSet(0)));
         }
         {
@@ -307,7 +307,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
 
             UNIT_ASSERT(result.IsSuccess());
             NKqp::CompareYson(R"([
-                [[6u];[0u];["/Root/Tenant2/Table2"]]
+                [[10u];[0u];["/Root/Tenant2/Table2"]]
             ])", FormatResultSetYson(result.GetResultSet(0)));
         }
     }
@@ -1521,11 +1521,12 @@ Y_UNIT_TEST_SUITE(SystemView) {
             UNIT_ASSERT_VALUES_EQUAL(entry.Type, ESchemeEntryType::Directory);
 
             auto children = result.GetChildren();
-            UNIT_ASSERT_VALUES_EQUAL(children.size(), 4);
-            UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, "Table0");
-            UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, "Tenant1");
-            UNIT_ASSERT_STRINGS_EQUAL(children[2].Name, "Tenant2");
-            UNIT_ASSERT_STRINGS_EQUAL(children[3].Name, ".sys");
+            UNIT_ASSERT_VALUES_EQUAL(children.size(), 5);
+            UNIT_ASSERT_STRINGS_EQUAL(children[0].Name, ".metadata");
+            UNIT_ASSERT_STRINGS_EQUAL(children[1].Name, "Table0");
+            UNIT_ASSERT_STRINGS_EQUAL(children[2].Name, "Tenant1");
+            UNIT_ASSERT_STRINGS_EQUAL(children[3].Name, "Tenant2");
+            UNIT_ASSERT_STRINGS_EQUAL(children[4].Name, ".sys");
         }
         {
             auto result = schemeClient.ListDirectory("/Root/Tenant1").GetValueSync();
diff --git a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp
index 518740f0d9b..84d00cb0760 100644
--- a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp
@@ -3081,10 +3081,17 @@ Y_UNIT_TEST_SUITE(Cdc) {
     }
 
     Y_UNIT_TEST(EnqueueRequestProcessSend) {
+        // Disable workload manager because it's also used events from ES_PRIVATE
+        // which squash with event NChangeExchange::TEvChangeExchangePrivate::EvReady in test
+        NKikimrConfig::TAppConfig config;
+        config.MutableFeatureFlags()->SetEnableResourcePools(false);
+
         TPortManager portManager;
         TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig())
             .SetUseRealThreads(false)
             .SetDomainName("Root")
+            .SetEnableResourcePools(false)
+            .SetAppConfig(config)
         );
 
         auto& runtime = *server->GetRuntime();
diff --git a/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp b/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp
index bbfad76c801..9c6a817612f 100644
--- a/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_kqp_errors.cpp
@@ -34,13 +34,15 @@ bool HasIssue(const TIssues& issues, ui32 code, TStringBuf message, std::functio
 
 class TLocalFixture {
 public:
-    TLocalFixture() {
+    TLocalFixture(bool enableResourcePools = true) {
         TPortManager pm;
         NKikimrConfig::TAppConfig app;
+        app.MutableFeatureFlags()->SetEnableResourcePools(enableResourcePools);
         TServerSettings serverSettings(pm.GetPort(2134));
         serverSettings.SetDomainName("Root")
             .SetNodeCount(2)
             .SetUseRealThreads(false)
+            .SetEnableResourcePools(enableResourcePools)
             .SetAppConfig(app);
 
         Server = new TServer(serverSettings);
@@ -82,7 +84,8 @@ public:
 Y_UNIT_TEST_SUITE(KqpErrors) {
 
 Y_UNIT_TEST(ResolveTableError) {
-    TLocalFixture fixture;
+    // Disable resource pool, because workload manager also got TEvNavigateKeySetResult for default pool creation
+    TLocalFixture fixture(false);
     auto mitm = [&](TAutoPtr<IEventHandle> &ev) {
         if (ev->GetTypeRewrite() == TEvTxProxySchemeCache::TEvNavigateKeySetResult::EventType) {
             auto event = ev.Get()->Get<TEvTxProxySchemeCache::TEvNavigateKeySetResult>();
diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp
index 61892023837..19276401dac 100644
--- a/ydb/core/viewer/viewer_ut.cpp
+++ b/ydb/core/viewer/viewer_ut.cpp
@@ -409,7 +409,11 @@ Y_UNIT_TEST_SUITE(Viewer) {
                 case TEvTxProxySchemeCache::EvNavigateKeySetResult: {
                     auto *x = reinterpret_cast<TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr*>(&ev);
                     auto &domain = (*x)->Get()->Request->ResultSet.begin()->DomainInfo;
-                    domain->Params.SetHive(1);
+
+                    // Event can be generated by workload manager, it is ok for not found response without domain
+                    if (domain) {
+                        domain->Params.SetHive(1);
+                    }
                     break;
                 }
                 case TEvHive::EvResponseHiveDomainStats: {
diff --git a/ydb/services/ydb/ydb_table_split_ut.cpp b/ydb/services/ydb/ydb_table_split_ut.cpp
index a66a899ea93..617351967b8 100644
--- a/ydb/services/ydb/ydb_table_split_ut.cpp
+++ b/ydb/services/ydb/ydb_table_split_ut.cpp
@@ -436,7 +436,10 @@ Y_UNIT_TEST_SUITE(YdbTableSplit) {
         TIntrusivePtr<TTestTimeProvider> testTimeProvider = new TTestTimeProvider(originalTimeProvider);
         NKikimr::TAppData::TimeProvider = testTimeProvider;
 
-        TKikimrWithGrpcAndRootSchemaNoSystemViews server;
+        NKikimrConfig::TAppConfig appConfig;
+        appConfig.MutableFeatureFlags()->SetEnableResourcePools(true);
+
+        TKikimrWithGrpcAndRootSchemaNoSystemViews server(appConfig);
         server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_NOTICE);
         server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::GRPC_SERVER, NActors::NLog::PRI_NOTICE);
         server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::TX_PROXY, NActors::NLog::PRI_NOTICE);
@@ -586,8 +589,12 @@ Y_UNIT_TEST_SUITE(YdbTableSplit) {
             UNIT_ASSERT_EQUAL(entry.Type, NYdb::NScheme::ESchemeEntryType::Directory);
 
             auto children = val.GetChildren();
-            UNIT_ASSERT_EQUAL_C(children.size(), 1, children.size());
+            UNIT_ASSERT_EQUAL_C(children.size(), 2, children.size());
             for (const auto& child: children) {
+                if (child.Name == ".metadata") {
+                    continue;
+                }
+
                 UNIT_ASSERT_EQUAL(child.Type, NYdb::NScheme::ESchemeEntryType::Table);
 
                 auto result = session.DropTable(TStringBuilder() << "Root" << "/" <<  child.Name).ExtractValueSync();
diff --git a/ydb/tests/functional/ydb_cli/test_ydb_backup.py b/ydb/tests/functional/ydb_cli/test_ydb_backup.py
index 32c4fbee068..4e155d49c2f 100644
--- a/ydb/tests/functional/ydb_cli/test_ydb_backup.py
+++ b/ydb/tests/functional/ydb_cli/test_ydb_backup.py
@@ -2,6 +2,7 @@
 
 from ydb.tests.library.common import yatest_common
 from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
+from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
 from ydb.tests.oss.ydb_sdk_import import ydb
 
 from hamcrest import assert_that, is_, is_not, contains_inanyorder, has_items
@@ -143,7 +144,7 @@ def list_all_dirs(prefix, path=""):
 class BaseTestBackupInFiles(object):
     @classmethod
     def setup_class(cls):
-        cls.cluster = kikimr_cluster_factory()
+        cls.cluster = kikimr_cluster_factory(KikimrConfigGenerator(extra_feature_flags=["enable_resource_pools"]))
         cls.cluster.start()
         cls.root_dir = "/Root"
         driver_config = ydb.DriverConfig(
@@ -213,7 +214,7 @@ class TestBackupSingle(BaseTestBackupInFiles):
 
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["table", ".sys"])
+            contains_inanyorder("table", ".metadata", ".sys")
         )
 
 
@@ -229,7 +230,7 @@ class TestBackupSingleNotNull(BaseTestBackupInFiles):
 
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["table", ".sys"])
+            contains_inanyorder("table", ".metadata", ".sys")
         )
 
 
@@ -428,7 +429,7 @@ class TestSingleBackupRestore(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["folder", ".sys"])
+            contains_inanyorder("folder", ".metadata", ".sys")
         )
 
         # Restore table
@@ -447,7 +448,7 @@ class TestSingleBackupRestore(BaseTestBackupInFiles):
 
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            contains_inanyorder("folder", "restored" + postfix, ".sys")
+            contains_inanyorder("folder", "restored" + postfix, ".metadata", ".sys")
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root/restored" + postfix).children],
@@ -491,7 +492,7 @@ class TestBackupRestoreInRoot(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["folder", ".sys"])
+            contains_inanyorder("folder", ".metadata", ".sys")
         )
 
         # Restore table
@@ -508,7 +509,7 @@ class TestBackupRestoreInRoot(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            contains_inanyorder("folder", "table", ".sys")
+            contains_inanyorder("folder", "table", ".metadata", ".sys")
         )
         assert_that(
             is_tables_the_same(session, "/Root/folder/table", "/Root/table"),
@@ -547,7 +548,7 @@ class TestBackupRestoreInRootSchemeOnly(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["folder", ".sys"])
+            contains_inanyorder("folder", ".metadata", ".sys")
         )
 
         # Restore table
@@ -564,7 +565,7 @@ class TestBackupRestoreInRootSchemeOnly(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            contains_inanyorder("folder", "table", ".sys")
+            contains_inanyorder("folder", "table", ".metadata", ".sys")
         )
         assert_that(
             is_tables_the_same(session, "/Root/folder/table", "/Root/table", False),
@@ -601,7 +602,7 @@ class TestIncompleteBackup(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["folder", ".sys"])
+            contains_inanyorder("folder", ".metadata", ".sys")
         )
 
         # Create "incomplete" file in folder with backup files
@@ -627,7 +628,7 @@ class TestIncompleteBackup(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["folder", ".sys"])
+            contains_inanyorder("folder", ".metadata", ".sys")
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root/folder").children],
@@ -653,7 +654,7 @@ class TestIncompleteBackup(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["folder", ".sys"])
+            contains_inanyorder("folder", ".metadata", ".sys")
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root/folder").children],
@@ -717,7 +718,7 @@ class TestAlterBackupRestore(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            is_(["folder", ".sys"])
+            contains_inanyorder("folder", ".metadata", ".sys")
         )
 
         # Restore table
@@ -734,7 +735,7 @@ class TestAlterBackupRestore(BaseTestBackupInFiles):
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root").children],
-            contains_inanyorder("folder", "restored", ".sys")
+            contains_inanyorder("folder", "restored", ".metadata", ".sys")
         )
         assert_that(
             [child.name for child in self.driver.scheme_client.list_directory("/Root/restored").children],