aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorOleg Doronin <dorooleg@yandex.ru>2024-12-20 13:48:31 +0100
committerGitHub <noreply@github.com>2024-12-20 15:48:31 +0300
commita580b00ce41af1f7a6d81041a4e87cde9c5cc665 (patch)
treeea82ba27038ffbf9f973e98a3d2457ec4388177e
parentd30aab14a644965dd83df55c3d2a9f3424fcdf9b (diff)
downloadydb-a580b00ce41af1f7a6d81041a4e87cde9c5cc665.tar.gz
Supported the ability to grant access to external data sources (#10848)
-rw-r--r--ydb/core/fq/libs/compute/common/config.h59
-rw-r--r--ydb/core/fq/libs/config/protos/compute.proto5
-rw-r--r--ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp113
-rw-r--r--ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h5
-rw-r--r--ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp26
-rw-r--r--ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h1
-rw-r--r--ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp1
7 files changed, 140 insertions, 70 deletions
diff --git a/ydb/core/fq/libs/compute/common/config.h b/ydb/core/fq/libs/compute/common/config.h
index d113fa7837..579869553e 100644
--- a/ydb/core/fq/libs/compute/common/config.h
+++ b/ydb/core/fq/libs/compute/common/config.h
@@ -68,6 +68,25 @@ public:
}
}
+ TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope) const {
+ const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
+ switch (controlPlane.type_case()) {
+ case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET:
+ return {};
+ case NConfig::TYdbComputeControlPlane::kSingle:
+ return {};
+ case NConfig::TYdbComputeControlPlane::kCms:
+ return GetExternalSourcesAccessSIDs(scope, controlPlane.GetCms().GetDatabaseMapping());
+ case NConfig::TYdbComputeControlPlane::kYdbcp:
+ return GetExternalSourcesAccessSIDs(scope, controlPlane.GetYdbcp().GetDatabaseMapping());
+ }
+ }
+
+ TVector<TString> GetExternalSourcesAccessSIDs(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
+ const auto protoExternalSourcesAccessSIDs = GetComputeDatabaseConfig(scope, databaseMapping).GetAccessConfig().GetExternalSourcesAccessSID();
+ return TVector<TString>{protoExternalSourcesAccessSIDs.begin(), protoExternalSourcesAccessSIDs.end()};
+ }
+
NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope) const {
const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
switch (controlPlane.type_case()) {
@@ -83,15 +102,8 @@ public:
}
NFq::NConfig::TYdbStorageConfig GetControlPlaneConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
- auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
- if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
- return it->second.GetControlPlaneConnection();
- }
- return databaseMapping.GetCommon().empty()
- ? NFq::NConfig::TYdbStorageConfig{}
- : databaseMapping
- .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
- .GetControlPlaneConnection();
+ auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
+ return computeDatabaseConfig.GetControlPlaneConnection();
}
NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope) const {
@@ -137,18 +149,11 @@ public:
}
NFq::NConfig::TYdbStorageConfig GetExecutionConnection(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
- auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
- if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
- return it->second.GetExecutionConnection();
- }
- return databaseMapping.GetCommon().empty()
- ? NFq::NConfig::TYdbStorageConfig{}
- : databaseMapping
- .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
- .GetExecutionConnection();
+ auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
+ return computeDatabaseConfig.GetExecutionConnection();
}
- NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) {
+ NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope) const {
const auto& controlPlane = ComputeConfig.GetYdb().GetControlPlane();
switch (controlPlane.type_case()) {
case NConfig::TYdbComputeControlPlane::TYPE_NOT_SET:
@@ -163,15 +168,19 @@ public:
}
NFq::NConfig::TWorkloadManagerConfig GetWorkloadManagerConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
+ auto computeDatabaseConfig = GetComputeDatabaseConfig(scope, databaseMapping);
+ return computeDatabaseConfig.GetWorkloadManagerConfig();
+ }
+
+ NFq::NConfig::TComputeDatabaseConfig GetComputeDatabaseConfig(const TString& scope, const ::NFq::NConfig::TDatabaseMapping& databaseMapping) const {
auto it = databaseMapping.GetScopeToComputeDatabase().find(scope);
if (it != databaseMapping.GetScopeToComputeDatabase().end()) {
- return it->second.GetWorkloadManagerConfig();
+ return it->second;
+ }
+ if (databaseMapping.GetCommon().empty()) {
+ return NFq::NConfig::TComputeDatabaseConfig{};
}
- return databaseMapping.GetCommon().empty()
- ? NFq::NConfig::TWorkloadManagerConfig{}
- : databaseMapping
- .GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size())
- .GetWorkloadManagerConfig();
+ return databaseMapping.GetCommon(MultiHash(scope) % databaseMapping.GetCommon().size());
}
bool YdbComputeControlPlaneEnabled(const TString& scope) const {
diff --git a/ydb/core/fq/libs/config/protos/compute.proto b/ydb/core/fq/libs/config/protos/compute.proto
index 7ce620b59e..72b6826b69 100644
--- a/ydb/core/fq/libs/config/protos/compute.proto
+++ b/ydb/core/fq/libs/config/protos/compute.proto
@@ -42,6 +42,10 @@ message TWorkloadManagerConfig {
repeated TResourcePool ResourcePool = 3;
}
+message TAccessConfig {
+ repeated string ExternalSourcesAccessSID = 1;
+}
+
message TComputeDatabaseConfig {
string Id = 7;
TYdbStorageConfig ControlPlaneConnection = 1;
@@ -50,6 +54,7 @@ message TComputeDatabaseConfig {
string Tenant = 2;
TLoadControlConfig LoadControlConfig = 4;
TWorkloadManagerConfig WorkloadManagerConfig = 5;
+ TAccessConfig AccessConfig = 8;
}
message TDatabaseMapping {
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp
index f842ccf6a3..db162d81f5 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.cpp
@@ -17,6 +17,61 @@ TString MakeSecretKeyName(const TString& prefix, const TString& folderId, const
return TStringBuilder{} << prefix << "_" << folderId << "_" << name;
}
+TString MakeCreateSecretObjectSql(const TString& name, const TString& value) {
+ using namespace fmt::literals;
+ return fmt::format(
+ R"(
+ UPSERT OBJECT {name} (TYPE SECRET) WITH value={value};
+ )",
+ "name"_a = EncloseAndEscapeString(name, '`'),
+ "value"_a = EncloseSecret(EncloseAndEscapeString(value, '"')));
+}
+
+TString MakeCreateSecretAccessObjectSql(const TString& name, const TString& sid) {
+ using namespace fmt::literals;
+ TString accessName = TStringBuilder{} << name << ":" << sid;
+ return fmt::format(
+ R"(
+ UPSERT OBJECT {name} (TYPE SECRET_ACCESS);
+ )",
+ "name"_a = EncloseAndEscapeString(accessName, '`'));
+}
+
+TString MakeCreateSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) {
+ TStringBuilder result;
+ for (const auto& sid : externalSourcesAccessSIDs) {
+ result << MakeCreateSecretAccessObjectSql(name, sid);
+ }
+ return result;
+}
+
+TString MakeDropSecretObjectSql(const TString& name) {
+ using namespace fmt::literals;
+ return fmt::format(
+ R"(
+ DROP OBJECT {name} (TYPE SECRET);
+ )",
+ "name"_a = EncloseAndEscapeString(name, '`'));
+}
+
+TString MakeDropAccessSecretObjectSql(const TString& name, const TString& sid) {
+ using namespace fmt::literals;
+ TString accessName = TStringBuilder{} << name << ":" << sid;
+ return fmt::format(
+ R"(
+ DROP OBJECT {name} (TYPE SECRET_ACCESS);
+ )",
+ "name"_a = EncloseAndEscapeString(accessName, '`'));
+}
+
+TString MakeDropSecretAccessObjectsSql(const TString& name, const TVector<TString>& externalSourcesAccessSIDs) {
+ TStringBuilder result;
+ for (const auto& sid : externalSourcesAccessSIDs) {
+ result << MakeDropAccessSecretObjectSql(name, sid);
+ }
+ return result;
+}
+
}
TString MakeCreateExternalDataTableQuery(const FederatedQuery::BindingContent& content,
@@ -100,33 +155,30 @@ TString SignAccountId(const TString& id, const TSigner::TPtr& signer) {
return signer ? signer->SignAccountId(id) : TString{};
}
+
+
TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting,
- const TString& name,
- const TSigner::TPtr& signer,
- const TString& folderId) {
- using namespace fmt::literals;
- TString secretObjects;
+ const TString& name,
+ const TSigner::TPtr& signer,
+ const TString& folderId,
+ const TVector<TString>& externalSourcesAccessSIDs) {
+ TStringBuilder result;
auto serviceAccountId = ExtractServiceAccountId(setting);
- if (serviceAccountId) {
- secretObjects = signer ? fmt::format(
- R"(
- UPSERT OBJECT {sa_secret_name} (TYPE SECRET) WITH value={signature};
- )",
- "sa_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'),
- "signature"_a = EncloseSecret(EncloseAndEscapeString(SignAccountId(serviceAccountId, signer), '"'))) : std::string{};
+ if (serviceAccountId && signer) {
+ const TString saSecretName = MakeSecretKeyName("f1", folderId, name);
+ const TString signature = SignAccountId(serviceAccountId, signer);
+ result << MakeCreateSecretObjectSql(saSecretName, signature);
+ result << MakeCreateSecretAccessObjectsSql(saSecretName, externalSourcesAccessSIDs);
}
auto password = GetPassword(setting);
if (password) {
- secretObjects += fmt::format(
- R"(
- UPSERT OBJECT {password_secret_name} (TYPE SECRET) WITH value={password};
- )",
- "password_secret_name"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'),
- "password"_a = EncloseSecret(EncloseAndEscapeString(*password, '"')));
+ const TString passwordSecretName = MakeSecretKeyName("f2", folderId, name);
+ result << MakeCreateSecretObjectSql(passwordSecretName, *password);
+ result << MakeCreateSecretAccessObjectsSql(passwordSecretName, externalSourcesAccessSIDs);
}
- return secretObjects ? secretObjects : TMaybe<TString>{};
+ return result ? result : TMaybe<TString>{};
}
TString CreateAuthParamsQuery(const FederatedQuery::ConnectionSetting& setting,
@@ -301,21 +353,14 @@ TString MakeCreateExternalDataSourceQuery(
folderId));
}
-TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId) {
- using namespace fmt::literals;
- return fmt::format(
- R"(
- DROP OBJECT {secret_name1} (TYPE SECRET);
- DROP OBJECT {secret_name2} (TYPE SECRET);
- DROP OBJECT {secret_name3} (TYPE SECRET); -- for backward compatibility
- DROP OBJECT {secret_name4} (TYPE SECRET); -- for backward compatibility
- DROP OBJECT {secret_name5} (TYPE SECRET); -- for backward compatibility
- )",
- "secret_name1"_a = EncloseAndEscapeString(MakeSecretKeyName("f1", folderId, name), '`'),
- "secret_name2"_a = EncloseAndEscapeString(MakeSecretKeyName("f2", folderId, name), '`'),
- "secret_name3"_a = EncloseAndEscapeString(TStringBuilder{} << "k1" << name, '`'),
- "secret_name4"_a = EncloseAndEscapeString(TStringBuilder{} << "k2" << name, '`'),
- "secret_name5"_a = EncloseAndEscapeString(name, '`'));
+TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs) {
+ const TString secretName1 = MakeSecretKeyName("f1", folderId, name);
+ const TString secretName2 = MakeSecretKeyName("f2", folderId, name);
+ return TStringBuilder{}
+ << MakeDropSecretAccessObjectsSql(secretName1, externalSourcesAccessSIDs)
+ << MakeDropSecretObjectSql(secretName1)
+ << MakeDropSecretAccessObjectsSql(secretName2, externalSourcesAccessSIDs)
+ << MakeDropSecretObjectSql(secretName2);
}
TString MakeDeleteExternalDataTableQuery(const TString& tableName) {
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h
index 92ed74341c..5d8e86df13 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/query_utils.h
@@ -11,9 +11,10 @@ namespace NPrivate {
TMaybe<TString> CreateSecretObjectQuery(const FederatedQuery::ConnectionSetting& setting,
const TString& name,
const TSigner::TPtr& signer,
- const TString& folderId);
+ const TString& folderId,
+ const TVector<TString>& externalSourcesAccessSIDs);
-TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId);
+TMaybe<TString> DropSecretObjectQuery(const TString& name, const TString& folderId, const TVector<TString>& externalSourcesAccessSIDs);
TString MakeCreateExternalDataSourceQuery(
const FederatedQuery::ConnectionContent& connectionContent,
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp
index 26eae1ef76..e62849f7da 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.cpp
@@ -564,7 +564,8 @@ IActor* MakeCreateConnectionActor(
auto createSecretStatement = CreateSecretObjectQuery(connectionContent.setting(),
connectionContent.name(),
signer,
- folderId);
+ folderId,
+ computeConfig.GetExternalSourcesAccessSIDs(scope));
std::vector<TSchemaQueryTask> statements;
if (createSecretStatement) {
@@ -665,14 +666,16 @@ IActor* MakeModifyConnectionActor(
auto& newConnectionContent = request->Get()->Request.content();
const auto& scope = request->Get()->Scope;
const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder();
+ const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope);
auto dropOldSecret =
- DropSecretObjectQuery(oldConnectionContent.name(), folderId);
+ DropSecretObjectQuery(oldConnectionContent.name(), folderId, externalSourcesAccessSIDs);
auto createNewSecret =
CreateSecretObjectQuery(newConnectionContent.setting(),
newConnectionContent.name(),
signer,
- folderId);
+ folderId,
+ externalSourcesAccessSIDs);
bool replaceSupported = computeConfig.IsReplaceIfExistsSyntaxSupported();
if (replaceSupported &&
@@ -680,7 +683,8 @@ IActor* MakeModifyConnectionActor(
// CREATE OR REPLACE
auto createSecretStatement =
CreateSecretObjectQuery(newConnectionContent.setting(),
- newConnectionContent.name(), signer, folderId);
+ newConnectionContent.name(), signer, folderId,
+ externalSourcesAccessSIDs);
std::vector<TSchemaQueryTask> statements;
if (createSecretStatement) {
@@ -727,13 +731,14 @@ IActor* MakeModifyConnectionActor(
.SQL = *dropOldSecret,
.RollbackSQL = CreateSecretObjectQuery(oldConnectionContent.setting(),
oldConnectionContent.name(),
- signer, folderId),
+ signer, folderId,
+ externalSourcesAccessSIDs),
.ShouldSkipStepOnError = IsPathDoesNotExistIssue});
}
if (createNewSecret) {
statements.push_back(TSchemaQueryTask{.SQL = *createNewSecret,
.RollbackSQL = DropSecretObjectQuery(
- newConnectionContent.name(), folderId)});
+ newConnectionContent.name(), folderId, externalSourcesAccessSIDs)});
}
statements.push_back(
@@ -787,18 +792,20 @@ IActor* MakeDeleteConnectionActor(
TDuration requestTimeout,
TCounters& counters,
const TCommonConfig& commonConfig,
+ const ::NFq::TComputeConfig& computeConfig,
TSigner::TPtr signer) {
auto queryFactoryMethod =
[signer = std::move(signer),
- commonConfig](
+ commonConfig, computeConfig](
const TEvControlPlaneProxy::TEvDeleteConnectionRequest::TPtr& request)
-> std::vector<TSchemaQueryTask> {
auto& connectionContent = *request->Get()->ConnectionContent;
const auto& scope = request->Get()->Scope;
const TString folderId = NYdb::NFq::TScope{scope}.ParseFolder();
+ const auto externalSourcesAccessSIDs = computeConfig.GetExternalSourcesAccessSIDs(scope);
auto dropSecret =
- DropSecretObjectQuery(connectionContent.name(), folderId);
+ DropSecretObjectQuery(connectionContent.name(), folderId, externalSourcesAccessSIDs);
std::vector statements = {
TSchemaQueryTask{.SQL = TString{MakeDeleteExternalDataSourceQuery(
@@ -812,7 +819,8 @@ IActor* MakeDeleteConnectionActor(
.RollbackSQL =
CreateSecretObjectQuery(connectionContent.setting(),
connectionContent.name(),
- signer, folderId),
+ signer, folderId,
+ externalSourcesAccessSIDs),
.ShouldSkipStepOnError = IsPathDoesNotExistIssue});
}
return statements;
diff --git a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h
index 2be77b8440..260c8ecaaf 100644
--- a/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h
+++ b/ydb/core/fq/libs/control_plane_proxy/actors/ydb_schema_query_actor.h
@@ -45,6 +45,7 @@ NActors::IActor* MakeDeleteConnectionActor(
TDuration requestTimeout,
TCounters& counters,
const NConfig::TCommonConfig& commonConfig,
+ const TComputeConfig& computeConfig,
TSigner::TPtr signer);
/// Binding manipulation actors
diff --git a/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp b/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp
index cac729cf17..e79fee68ad 100644
--- a/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp
+++ b/ydb/core/fq/libs/control_plane_proxy/control_plane_proxy.cpp
@@ -1872,6 +1872,7 @@ private:
Config.RequestTimeout,
Counters,
Config.CommonConfig,
+ Config.ComputeConfig,
Signer));
return;
}