diff options
author | Oleg Doronin <dorooleg@yandex.ru> | 2024-12-20 13:48:31 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-12-20 15:48:31 +0300 |
commit | a580b00ce41af1f7a6d81041a4e87cde9c5cc665 (patch) | |
tree | ea82ba27038ffbf9f973e98a3d2457ec4388177e | |
parent | d30aab14a644965dd83df55c3d2a9f3424fcdf9b (diff) | |
download | ydb-a580b00ce41af1f7a6d81041a4e87cde9c5cc665.tar.gz |
Supported the ability to grant access to external data sources (#10848)
7 files changed, 140 insertions, 70 deletions
diff --git a/ydb/core/fq/libs/compute/common/config.h b/ydb/core/fq/libs/compute/common/config.h index d113fa7837..579869553e 100644 --- a/ydb/core/fq/libs/compute/common/config.h +++ b/ydb/core/fq/libs/compute/common/config.h @@ -68,6 +68,25 @@ public: } } + TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope) const { + const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane(); + switch (controlPlane.type_case()) { + case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET: + return {}; + case NConfig::TYdbComputeControlPlane::kSingle: + return {}; + case NConfig::TYdbComputeControlPlane::kCms: + return GetExternalSourcesAccessSIDs(scope, controlPlane.GetCms().GetDatabaseMapping()); + case NConfig::TYdbComputeControlPlane::kYdbcp: + return GetExternalSourcesAccessSIDs(scope, controlPlane.GetYdbcp().GetDatabaseMapping()); + } + } + + TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const { + const auto protoExternalSourcesAccessSIDs = GetComputeDatabaseConfig(scope, databaseMapping).GetAccessConfig().GetExternalSourcesAccessSID(); + return TVector<TString>{protoExternalSourcesAccessSIDs.begin(), protoExternalSourcesAccessSIDs.end()}; + } + NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope) const { const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane(); switch (controlPlane.type_case()) { @@ -83,15 +102,8 @@ public: } NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const { - auto it = databaseMapping.GetScopeToComputeDatabase().find(scope); - if (it != databaseMapping.GetScopeToComputeDatabase().end()) { - return it->second.GetControlPlaneConnection(); - } - return databaseMapping.GetCommon().empty() - ? NFq::NConfig::TYdbStorageConfig{} - : databaseMapping - .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size()) - .GetControlPlaneConnection(); + auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping); + return computeDatabaseConfig.GetControlPlaneConnection(); } NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope) const { @@ -137,18 +149,11 @@ public: } NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const { - auto it = databaseMapping.GetScopeToComputeDatabase().find(scope); - if (it != databaseMapping.GetScopeToComputeDatabase().end()) { - return it->second.GetExecutionConnection(); - } - return databaseMapping.GetCommon().empty() - ? NFq::NConfig::TYdbStorageConfig{} - : databaseMapping - .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size()) - .GetExecutionConnection(); + auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping); + return computeDatabaseConfig.GetExecutionConnection(); } - NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) { + NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) const { const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane(); switch (controlPlane.type_case()) { case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET: @@ -163,15 +168,19 @@ public: } NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const { + auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping); + return computeDatabaseConfig.GetWorkloadManagerConfig(); + } + + NFq::NConfig::TComputeDatabaseConfig GetComputeDatabaseConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const { auto it = databaseMapping.GetScopeToComputeDatabase().find(scope); if (it != databaseMapping.GetScopeToComputeDatabase().end()) { - return it->second.GetWorkloadManagerConfig(); + return it->second; + } + if (databaseMapping.GetCommon().empty()) { + return NFq::NConfig::TComputeDatabaseConfig{}; } - return databaseMapping.GetCommon().empty() - ? NFq::NConfig::TWorkloadManagerConfig{} - : databaseMapping - .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size()) - .GetWorkloadManagerConfig(); + return databaseMapping.GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size()); } bool YdbComputeControlPlaneEnabled(const TString& scope) const { diff --git a/ydb/core/fq/libs/config/protos/compute.proto b/ydb/core/fq/libs/config/protos/compute.proto index 7ce620b59e..72b6826b69 100644 --- a/ydb/core/fq/libs/config/protos/compute.proto +++ b/ydb/core/fq/libs/config/protos/compute.proto @@ -42,6 +42,10 @@ message TWorkloadManagerConfig { repeated TResourcePool ResourcePool = 3; } +message TAccessConfig { + repeated string ExternalSourcesAccessSID = 1; +} + message TComputeDatabaseConfig { string Id = 7; TYdbStorageConfig ControlPlaneConnection = 1; @@ -50,6 +54,7 @@ message TComputeDatabaseConfig { string Tenant = 2; TLoadControlConfig LoadControlConfig = 4; TWorkloadManagerConfig WorkloadManagerConfig = 5; + TAccessConfig AccessConfig = 8; } message TDatabaseMapping { diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp index f842ccf6a3..db162d81f5 100644 --- a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp +++ b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp @@ -17,6 +17,61 @@ TString MakeSecretKeyName(const TString& prefix, const TString& folderId, const return TStringBuilder{} << prefix << "_" << folderId << "_" << name; } +TString MakeCreateSecretObjectSql(const TString& name, const TString& value) { + using namespace fmt::literals; + return fmt::format( + R"( + UPSERT OBJECT {name} (TYPE SECRET) WITH value={value}; + )", + "name"_a = EncloseAndEscapeString(name, '`'), + "value"_a = EncloseSecret(EncloseAndEscapeString(value, '"'))); +} + +TString MakeCreateSecretAccessObjectSql(const TString& name, const TString& sid) { + using namespace fmt::literals; + TString accessName = TStringBuilder{} << name << ":" << sid; + return fmt::format( + R"( + UPSERT OBJECT {name} (TYPE SECRET_ACCESS); + )", + "name"_a = EncloseAndEscapeString(accessName, '`')); +} + +TString MakeCreateSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) { + TStringBuilder result; + for (const auto& sid : externalSourcesAccessSIDs) { + result << MakeCreateSecretAccessObjectSql(name, sid); + } + return result; +} + +TString MakeDropSecretObjectSql(const TString& name) { + using namespace fmt::literals; + return fmt::format( + R"( + DROP OBJECT {name} (TYPE SECRET); + )", + "name"_a = EncloseAndEscapeString(name, '`')); +} + +TString MakeDropAccessSecretObjectSql(const TString& name, const TString& sid) { + using namespace fmt::literals; + TString accessName = TStringBuilder{} << name << ":" << sid; + return fmt::format( + R"( + DROP OBJECT {name} (TYPE SECRET_ACCESS); + )", + "name"_a = EncloseAndEscapeString(accessName, '`')); +} + +TString MakeDropSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) { + TStringBuilder result; + for (const auto& sid : externalSourcesAccessSIDs) { + result << MakeDropAccessSecretObjectSql(name, sid); + } + return result; +} + } TString MakeCreateExternalDataTableQuery(const FederatedQuery::BindingContent& content, @@ -100,33 +155,30 @@ TString SignAccountId(const TString& id, const TSigner::TPtr& signer) { return signer ? signer->SignAccountId(id) : TString{}; } + + TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting, - const TString& name, - const TSigner::TPtr& signer, - const TString& folderId) { - using namespace fmt::literals; - TString secretObjects; + const TString& name, + const TSigner::TPtr& signer, + const TString& folderId, + const TVector<TString>& externalSourcesAccessSIDs) { + TStringBuilder result; auto serviceAccountId = ExtractServiceAccountId(setting); - if (serviceAccountId) { - secretObjects = signer ? fmt::format( - R"( - UPSERT OBJECT {sa_secret_name} (TYPE SECRET) WITH value={signature}; - )", - "sa_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'), - "signature"_a = EncloseSecret(EncloseAndEscapeString(SignAccountId(serviceAccountId, signer), '"'))) : std::string{}; + if (serviceAccountId && signer) { + const TString saSecretName = MakeSecretKeyName("f1", folderId, name); + const TString signature = SignAccountId(serviceAccountId, signer); + result << MakeCreateSecretObjectSql(saSecretName, signature); + result << MakeCreateSecretAccessObjectsSql(saSecretName, externalSourcesAccessSIDs); } auto password = GetPassword(setting); if (password) { - secretObjects += fmt::format( - R"( - UPSERT OBJECT {password_secret_name} (TYPE SECRET) WITH value={password}; - )", - "password_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'), - "password"_a = EncloseSecret(EncloseAndEscapeString(*password, '"'))); + const TString passwordSecretName = MakeSecretKeyName("f2", folderId, name); + result << MakeCreateSecretObjectSql(passwordSecretName, *password); + result << MakeCreateSecretAccessObjectsSql(passwordSecretName, externalSourcesAccessSIDs); } - return secretObjects ? secretObjects : TMaybe<TString>{}; + return result ? result : TMaybe<TString>{}; } TString CreateAuthParamsQuery(const FederatedQuery::ConnectionSetting& setting, @@ -301,21 +353,14 @@ TString MakeCreateExternalDataSourceQuery( folderId)); } -TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId) { - using namespace fmt::literals; - return fmt::format( - R"( - DROP OBJECT {secret_name1} (TYPE SECRET); - DROP OBJECT {secret_name2} (TYPE SECRET); - DROP OBJECT {secret_name3} (TYPE SECRET); -- for backward compatibility - DROP OBJECT {secret_name4} (TYPE SECRET); -- for backward compatibility - DROP OBJECT {secret_name5} (TYPE SECRET); -- for backward compatibility - )", - "secret_name1"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'), - "secret_name2"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'), - "secret_name3"_a = EncloseAndEscapeString(TStringBuilder{} << "k1" << name, '`'), - "secret_name4"_a = EncloseAndEscapeString(TStringBuilder{} << "k2" << name, '`'), - "secret_name5"_a = EncloseAndEscapeString(name, '`')); +TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs) { + const TString secretName1 = MakeSecretKeyName("f1", folderId, name); + const TString secretName2 = MakeSecretKeyName("f2", folderId, name); + return TStringBuilder{} + << MakeDropSecretAccessObjectsSql(secretName1, externalSourcesAccessSIDs) + << MakeDropSecretObjectSql(secretName1) + << MakeDropSecretAccessObjectsSql(secretName2, externalSourcesAccessSIDs) + << MakeDropSecretObjectSql(secretName2); } TString MakeDeleteExternalDataTableQuery(const TString& tableName) { diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h index 92ed74341c..5d8e86df13 100644 --- a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h +++ b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h @@ -11,9 +11,10 @@ namespace NPrivate { TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting, const TString& name, const TSigner::TPtr& signer, - const TString& folderId); + const TString& folderId, + const TVector<TString>& externalSourcesAccessSIDs); -TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId); +TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs); TString MakeCreateExternalDataSourceQuery( const FederatedQuery::ConnectionContent& connectionContent, diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp index 26eae1ef76..e62849f7da 100644 --- a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp +++ b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp @@ -564,7 +564,8 @@ IActor* MakeCreateConnectionActor( auto createSecretStatement = CreateSecretObjectQuery(connectionContent.setting(), connectionContent.name(), signer, - folderId); + folderId, + computeConfig.GetExternalSourcesAccessSIDs(scope)); std::vector<TSchemaQueryTask> statements; if (createSecretStatement) { @@ -665,14 +666,16 @@ IActor* MakeModifyConnectionActor( auto& newConnectionContent = request->Get()->Request.content(); const auto& scope = request->Get()->Scope; const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder(); + const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope); auto dropOldSecret = - DropSecretObjectQuery(oldConnectionContent.name(), folderId); + DropSecretObjectQuery(oldConnectionContent.name(), folderId, externalSourcesAccessSIDs); auto createNewSecret = CreateSecretObjectQuery(newConnectionContent.setting(), newConnectionContent.name(), signer, - folderId); + folderId, + externalSourcesAccessSIDs); bool replaceSupported = computeConfig.IsReplaceIfExistsSyntaxSupported(); if (replaceSupported && @@ -680,7 +683,8 @@ IActor* MakeModifyConnectionActor( // CREATE OR REPLACE auto createSecretStatement = CreateSecretObjectQuery(newConnectionContent.setting(), - newConnectionContent.name(), signer, folderId); + newConnectionContent.name(), signer, folderId, + externalSourcesAccessSIDs); std::vector<TSchemaQueryTask> statements; if (createSecretStatement) { @@ -727,13 +731,14 @@ IActor* MakeModifyConnectionActor( .SQL = *dropOldSecret, .RollbackSQL = CreateSecretObjectQuery(oldConnectionContent.setting(), oldConnectionContent.name(), - signer, folderId), + signer, folderId, + externalSourcesAccessSIDs), .ShouldSkipStepOnError = IsPathDoesNotExistIssue}); } if (createNewSecret) { statements.push_back(TSchemaQueryTask{.SQL = *createNewSecret, .RollbackSQL = DropSecretObjectQuery( - newConnectionContent.name(), folderId)}); + newConnectionContent.name(), folderId, externalSourcesAccessSIDs)}); } statements.push_back( @@ -787,18 +792,20 @@ IActor* MakeDeleteConnectionActor( TDuration requestTimeout, TCounters& counters, const TCommonConfig& commonConfig, + const ::NFq::TComputeConfig& computeConfig, TSigner::TPtr signer) { auto queryFactoryMethod = [signer = std::move(signer), - commonConfig]( + commonConfig, computeConfig]( const TEvControlPlaneProxy::TEvDeleteConnectionRequest::TPtr& request) -> std::vector<TSchemaQueryTask> { auto& connectionContent = *request->Get()->ConnectionContent; const auto& scope = request->Get()->Scope; const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder(); + const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope); auto dropSecret = - DropSecretObjectQuery(connectionContent.name(), folderId); + DropSecretObjectQuery(connectionContent.name(), folderId, externalSourcesAccessSIDs); std::vector statements = { TSchemaQueryTask{.SQL = TString{MakeDeleteExternalDataSourceQuery( @@ -812,7 +819,8 @@ IActor* MakeDeleteConnectionActor( .RollbackSQL = CreateSecretObjectQuery(connectionContent.setting(), connectionContent.name(), - signer, folderId), + signer, folderId, + externalSourcesAccessSIDs), .ShouldSkipStepOnError = IsPathDoesNotExistIssue}); } return statements; diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h index 2be77b8440..260c8ecaaf 100644 --- a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h +++ b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h @@ -45,6 +45,7 @@ NActors::IActor* MakeDeleteConnectionActor( TDuration requestTimeout, TCounters& counters, const NConfig::TCommonConfig& commonConfig, + const TComputeConfig& computeConfig, TSigner::TPtr signer); /// Binding manipulation actors diff --git a/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp b/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp index cac729cf17..e79fee68ad 100644 --- a/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp +++ b/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp @@ -1872,6 +1872,7 @@ private: Config.RequestTimeout, Counters, Config.CommonConfig, + Config.ComputeConfig, Signer)); return; } |