Supported the ability to grant access to external data sources (#10848)

7 files changed, 140 insertions, 70 deletions

diff --git a/ydb/core/fq/libs/compute/common/config.h b/ydb/core/fq/libs/compute/common/config.h
index d113fa7837..579869553e 100644
--- a/ydb/core/fq/libs/compute/common/config.h
+++ b/ydb/core/fq/libs/compute/common/config.h
@@ -68,6 +68,25 @@ public:
         }
     }
 
+    TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope) const {
+        const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
+        switch (controlPlane.type_case()) {
+            case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET:
+                return {};
+            case NConfig::TYdbComputeControlPlane::kSingle:
+                return {};
+            case NConfig::TYdbComputeControlPlane::kCms:
+                return GetExternalSourcesAccessSIDs(scope, controlPlane.GetCms().GetDatabaseMapping());
+            case NConfig::TYdbComputeControlPlane::kYdbcp:
+                return GetExternalSourcesAccessSIDs(scope, controlPlane.GetYdbcp().GetDatabaseMapping());
+        }
+    }
+
+    TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
+        const auto protoExternalSourcesAccessSIDs = GetComputeDatabaseConfig(scope, databaseMapping).GetAccessConfig().GetExternalSourcesAccessSID();
+        return TVector<TString>{protoExternalSourcesAccessSIDs.begin(), protoExternalSourcesAccessSIDs.end()};
+    }
+
     NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope) const {
         const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
         switch (controlPlane.type_case()) {
@@ -83,15 +102,8 @@ public:
     }
 
     NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
-        auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
-        if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
-            return it->second.GetControlPlaneConnection();
-        }
-        return databaseMapping.GetCommon().empty()
-                   ? NFq::NConfig::TYdbStorageConfig{}
-                   : databaseMapping
-                         .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
-                         .GetControlPlaneConnection();
+        auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
+        return computeDatabaseConfig.GetControlPlaneConnection();
     }
 
     NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope) const {
@@ -137,18 +149,11 @@ public:
     }
 
     NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
-        auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
-        if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
-            return it->second.GetExecutionConnection();
-        }
-        return databaseMapping.GetCommon().empty()
-                   ? NFq::NConfig::TYdbStorageConfig{}
-                   : databaseMapping
-                         .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
-                         .GetExecutionConnection();
+        auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
+        return computeDatabaseConfig.GetExecutionConnection();
     }
 
-    NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) {
+    NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) const {
         const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
         switch (controlPlane.type_case()) {
             case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET:
@@ -163,15 +168,19 @@ public:
     }
 
     NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
+        auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
+        return computeDatabaseConfig.GetWorkloadManagerConfig();
+    }
+
+    NFq::NConfig::TComputeDatabaseConfig GetComputeDatabaseConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
         auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
         if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
-            return it->second.GetWorkloadManagerConfig();
+            return it->second;
+        }
+        if (databaseMapping.GetCommon().empty()) {
+            return NFq::NConfig::TComputeDatabaseConfig{};
         }
-        return databaseMapping.GetCommon().empty()
-                   ? NFq::NConfig::TWorkloadManagerConfig{}
-                   : databaseMapping
-                         .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
-                         .GetWorkloadManagerConfig();
+        return databaseMapping.GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size());
     }
 
     bool YdbComputeControlPlaneEnabled(const TString& scope) const {
diff --git a/ydb/core/fq/libs/config/protos/compute.proto b/ydb/core/fq/libs/config/protos/compute.proto
index 7ce620b59e..72b6826b69 100644
--- a/ydb/core/fq/libs/config/protos/compute.proto
+++ b/ydb/core/fq/libs/config/protos/compute.proto
@@ -42,6 +42,10 @@ message TWorkloadManagerConfig {
     repeated TResourcePool ResourcePool = 3;
 }
 
+message TAccessConfig {
+    repeated string ExternalSourcesAccessSID = 1;
+}
+
 message TComputeDatabaseConfig {
     string Id = 7;
     TYdbStorageConfig ControlPlaneConnection = 1;
@@ -50,6 +54,7 @@ message TComputeDatabaseConfig {
     string Tenant = 2;
     TLoadControlConfig LoadControlConfig = 4;
     TWorkloadManagerConfig WorkloadManagerConfig = 5;
+    TAccessConfig AccessConfig = 8;
 }
 
 message TDatabaseMapping {
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp
index f842ccf6a3..db162d81f5 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp
@@ -17,6 +17,61 @@ TString MakeSecretKeyName(const TString& prefix, const TString& folderId, const
     return TStringBuilder{} << prefix << "_" << folderId << "_" << name;
 }
 
+TString MakeCreateSecretObjectSql(const TString& name, const TString& value) {
+    using namespace fmt::literals;
+    return fmt::format(
+        R"(
+                UPSERT OBJECT {name} (TYPE SECRET) WITH value={value};
+            )",
+        "name"_a = EncloseAndEscapeString(name, '`'),
+        "value"_a = EncloseSecret(EncloseAndEscapeString(value, '"')));
+}
+
+TString MakeCreateSecretAccessObjectSql(const TString& name, const TString& sid) {
+    using namespace fmt::literals;
+    TString accessName = TStringBuilder{} << name << ":" << sid;
+    return fmt::format(
+        R"(
+            UPSERT OBJECT {name} (TYPE SECRET_ACCESS);
+        )",
+        "name"_a = EncloseAndEscapeString(accessName, '`'));
+}
+
+TString MakeCreateSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) {
+    TStringBuilder result;
+    for (const auto& sid : externalSourcesAccessSIDs) {
+        result << MakeCreateSecretAccessObjectSql(name, sid);
+    }
+    return result;
+}
+
+TString MakeDropSecretObjectSql(const TString& name) {
+    using namespace fmt::literals;
+    return fmt::format(
+        R"(
+                DROP OBJECT {name} (TYPE SECRET);
+            )",
+        "name"_a = EncloseAndEscapeString(name, '`'));
+}
+
+TString MakeDropAccessSecretObjectSql(const TString& name, const TString& sid) {
+    using namespace fmt::literals;
+    TString accessName = TStringBuilder{} << name << ":" << sid;
+    return fmt::format(
+        R"(
+            DROP OBJECT {name} (TYPE SECRET_ACCESS);
+        )",
+        "name"_a = EncloseAndEscapeString(accessName, '`'));
+}
+
+TString MakeDropSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) {
+    TStringBuilder result;
+    for (const auto& sid : externalSourcesAccessSIDs) {
+        result << MakeDropAccessSecretObjectSql(name, sid);
+    }
+    return result;
+}
+
 }
 
 TString MakeCreateExternalDataTableQuery(const FederatedQuery::BindingContent& content,
@@ -100,33 +155,30 @@ TString SignAccountId(const TString& id, const TSigner::TPtr& signer) {
     return signer ? signer->SignAccountId(id) : TString{};
 }
 
+
+
 TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting,
-                                const TString& name,
-                                const TSigner::TPtr& signer,
-                                const TString& folderId) {
-    using namespace fmt::literals;
-    TString secretObjects;
+                                        const TString& name,
+                                        const TSigner::TPtr& signer,
+                                        const TString& folderId,
+                                        const TVector<TString>& externalSourcesAccessSIDs) {
+    TStringBuilder result;
     auto serviceAccountId = ExtractServiceAccountId(setting);
-    if (serviceAccountId) {
-        secretObjects = signer ? fmt::format(
-            R"(
-                    UPSERT OBJECT {sa_secret_name} (TYPE SECRET) WITH value={signature};
-                )",
-            "sa_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'),
-            "signature"_a       = EncloseSecret(EncloseAndEscapeString(SignAccountId(serviceAccountId, signer), '"'))) : std::string{};
+    if (serviceAccountId && signer) {
+        const TString saSecretName = MakeSecretKeyName("f1", folderId, name);
+        const TString signature = SignAccountId(serviceAccountId, signer);
+        result << MakeCreateSecretObjectSql(saSecretName, signature);
+        result << MakeCreateSecretAccessObjectsSql(saSecretName, externalSourcesAccessSIDs);
     }
 
     auto password = GetPassword(setting);
     if (password) {
-        secretObjects += fmt::format(
-                R"(
-                    UPSERT OBJECT {password_secret_name} (TYPE SECRET) WITH value={password};
-                )",
-            "password_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'),
-            "password"_a = EncloseSecret(EncloseAndEscapeString(*password, '"')));
+        const TString passwordSecretName = MakeSecretKeyName("f2", folderId, name);
+        result << MakeCreateSecretObjectSql(passwordSecretName, *password);
+        result << MakeCreateSecretAccessObjectsSql(passwordSecretName, externalSourcesAccessSIDs);
     }
 
-    return secretObjects ? secretObjects : TMaybe<TString>{};
+    return result ? result : TMaybe<TString>{};
 }
 
 TString CreateAuthParamsQuery(const FederatedQuery::ConnectionSetting& setting,
@@ -301,21 +353,14 @@ TString MakeCreateExternalDataSourceQuery(
                                   folderId));
 }
 
-TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId) {
-    using namespace fmt::literals;
-    return fmt::format(
-            R"(
-                DROP OBJECT {secret_name1} (TYPE SECRET);
-                DROP OBJECT {secret_name2} (TYPE SECRET);
-                DROP OBJECT {secret_name3} (TYPE SECRET); -- for backward compatibility
-                DROP OBJECT {secret_name4} (TYPE SECRET); -- for backward compatibility
-                DROP OBJECT {secret_name5} (TYPE SECRET); -- for backward compatibility
-            )",
-        "secret_name1"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'),
-        "secret_name2"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'),
-        "secret_name3"_a = EncloseAndEscapeString(TStringBuilder{} << "k1" << name, '`'),
-        "secret_name4"_a = EncloseAndEscapeString(TStringBuilder{} << "k2" << name, '`'),
-        "secret_name5"_a = EncloseAndEscapeString(name, '`'));
+TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs) {
+    const TString secretName1 = MakeSecretKeyName("f1", folderId, name);
+    const TString secretName2 = MakeSecretKeyName("f2", folderId, name);
+    return TStringBuilder{}
+            << MakeDropSecretAccessObjectsSql(secretName1, externalSourcesAccessSIDs)
+            << MakeDropSecretObjectSql(secretName1)
+            << MakeDropSecretAccessObjectsSql(secretName2, externalSourcesAccessSIDs)
+            << MakeDropSecretObjectSql(secretName2);
 }
 
 TString MakeDeleteExternalDataTableQuery(const TString& tableName) {
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h
index 92ed74341c..5d8e86df13 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h
@@ -11,9 +11,10 @@ namespace NPrivate {
 TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting,
                                         const TString& name,
                                         const TSigner::TPtr& signer,
-                                        const TString& folderId);
+                                        const TString& folderId,
+                                        const TVector<TString>& externalSourcesAccessSIDs);
 
-TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId);
+TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs);
 
 TString MakeCreateExternalDataSourceQuery(
     const FederatedQuery::ConnectionContent& connectionContent,
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp
index 26eae1ef76..e62849f7da 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp
@@ -564,7 +564,8 @@ IActor* MakeCreateConnectionActor(
         auto createSecretStatement = CreateSecretObjectQuery(connectionContent.setting(),
                                                              connectionContent.name(),
                                                              signer,
-                                                             folderId);
+                                                             folderId,
+                                                             computeConfig.GetExternalSourcesAccessSIDs(scope));
 
         std::vector<TSchemaQueryTask> statements;
         if (createSecretStatement) {
@@ -665,14 +666,16 @@ IActor* MakeModifyConnectionActor(
         auto& newConnectionContent = request->Get()->Request.content();
         const auto& scope = request->Get()->Scope;
         const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder();
+        const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope);
 
         auto dropOldSecret =
-            DropSecretObjectQuery(oldConnectionContent.name(), folderId);
+            DropSecretObjectQuery(oldConnectionContent.name(), folderId, externalSourcesAccessSIDs);
         auto createNewSecret =
             CreateSecretObjectQuery(newConnectionContent.setting(),
                                     newConnectionContent.name(),
                                     signer,
-                                    folderId);
+                                    folderId,
+                                    externalSourcesAccessSIDs);
 
         bool replaceSupported = computeConfig.IsReplaceIfExistsSyntaxSupported();
         if (replaceSupported &&
@@ -680,7 +683,8 @@ IActor* MakeModifyConnectionActor(
             // CREATE OR REPLACE
             auto createSecretStatement =
                 CreateSecretObjectQuery(newConnectionContent.setting(),
-                                        newConnectionContent.name(), signer, folderId);
+                                        newConnectionContent.name(), signer, folderId, 
+                                        externalSourcesAccessSIDs);
 
             std::vector<TSchemaQueryTask> statements;
             if (createSecretStatement) {
@@ -727,13 +731,14 @@ IActor* MakeModifyConnectionActor(
                 .SQL         = *dropOldSecret,
                 .RollbackSQL = CreateSecretObjectQuery(oldConnectionContent.setting(),
                                                        oldConnectionContent.name(),
-                                                       signer, folderId),
+                                                       signer, folderId, 
+                                                       externalSourcesAccessSIDs),
                 .ShouldSkipStepOnError = IsPathDoesNotExistIssue});
         }
         if (createNewSecret) {
             statements.push_back(TSchemaQueryTask{.SQL         = *createNewSecret,
                                                   .RollbackSQL = DropSecretObjectQuery(
-                                                      newConnectionContent.name(), folderId)});
+                                                      newConnectionContent.name(), folderId, externalSourcesAccessSIDs)});
         }
 
         statements.push_back(
@@ -787,18 +792,20 @@ IActor* MakeDeleteConnectionActor(
     TDuration requestTimeout,
     TCounters& counters,
     const TCommonConfig& commonConfig,
+    const ::NFq::TComputeConfig& computeConfig,
     TSigner::TPtr signer) {
     auto queryFactoryMethod =
         [signer = std::move(signer),
-         commonConfig](
+         commonConfig, computeConfig](
             const TEvControlPlaneProxy::TEvDeleteConnectionRequest::TPtr& request)
         -> std::vector<TSchemaQueryTask> {
         auto& connectionContent = *request->Get()->ConnectionContent;
         const auto& scope = request->Get()->Scope;
         const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder();
+        const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope);
 
         auto dropSecret =
-            DropSecretObjectQuery(connectionContent.name(), folderId);
+            DropSecretObjectQuery(connectionContent.name(), folderId, externalSourcesAccessSIDs);
 
         std::vector statements = {
             TSchemaQueryTask{.SQL = TString{MakeDeleteExternalDataSourceQuery(
@@ -812,7 +819,8 @@ IActor* MakeDeleteConnectionActor(
                                  .RollbackSQL =
                                      CreateSecretObjectQuery(connectionContent.setting(),
                                                              connectionContent.name(),
-                                                             signer, folderId),
+                                                             signer, folderId,
+                                                             externalSourcesAccessSIDs),
                                  .ShouldSkipStepOnError = IsPathDoesNotExistIssue});
         }
         return statements;
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h
index 2be77b8440..260c8ecaaf 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h
@@ -45,6 +45,7 @@ NActors::IActor* MakeDeleteConnectionActor(
     TDuration requestTimeout,
     TCounters& counters,
     const NConfig::TCommonConfig& commonConfig,
+    const TComputeConfig& computeConfig,
     TSigner::TPtr signer);
 
 /// Binding manipulation actors
diff --git a/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp b/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp
index cac729cf17..e79fee68ad 100644
--- a/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp
+++ b/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp
@@ -1872,6 +1872,7 @@ private:
                                                    Config.RequestTimeout,
                                                    Counters,
                                                    Config.CommonConfig,
+                                                   Config.ComputeConfig,
                                                    Signer));
                 return;
             }