summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhcpp <[email protected]>2023-09-01 13:46:24 +0300
committerhcpp <[email protected]>2023-09-01 14:02:27 +0300
commitbf418335042882be6345a85cce0fa92e8c95537a (patch)
treebb3567b5bfd60b086f414a218e71df52d9e66d39
parentb7bb944233f923121a9fc3a7cdeaaaad25a9d6df (diff)
new data source properties have been added
-rw-r--r--ydb/core/external_sources/external_data_source.cpp23
-rw-r--r--ydb/core/external_sources/external_data_source.h3
-rw-r--r--ydb/core/external_sources/external_source.h6
-rw-r--r--ydb/core/external_sources/external_source_factory.cpp4
-rw-r--r--ydb/core/external_sources/object_storage.cpp12
-rw-r--r--ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp17
-rw-r--r--ydb/core/kqp/gateway/kqp_metadata_loader.cpp1
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_datasource.cpp5
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway.h1
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp46
-rw-r--r--ydb/core/protos/flat_scheme_op.proto5
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__init.cpp2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp13
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.cpp3
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_info_types.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path_describer.cpp1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_schema.h3
-rw-r--r--ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp28
-rw-r--r--ydb/library/yql/sql/v1/sql_translation.cpp14
-rw-r--r--ydb/library/yql/sql/v1/sql_ut.cpp17
-rw-r--r--ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema8
21 files changed, 172 insertions, 41 deletions
diff --git a/ydb/core/external_sources/external_data_source.cpp b/ydb/core/external_sources/external_data_source.cpp
index f0e11708535..e35a2643fa6 100644
--- a/ydb/core/external_sources/external_data_source.cpp
+++ b/ydb/core/external_sources/external_data_source.cpp
@@ -1,13 +1,15 @@
#include "external_data_source.h"
+#include <ydb/core/protos/flat_scheme_op.pb.h>
namespace NKikimr::NExternalSource {
namespace {
struct TExternalDataSource : public IExternalSource {
- TExternalDataSource(const TString& name, const TVector<TString>& authMethods)
+ TExternalDataSource(const TString& name, const TVector<TString>& authMethods, const TSet<TString>& availableProperties)
: Name(name)
, AuthMethods(authMethods)
+ , AvailableProperties(availableProperties)
{}
virtual TString Pack(const NKikimrExternalSources::TSchema&,
@@ -30,15 +32,30 @@ struct TExternalDataSource : public IExternalSource {
virtual TMap<TString, TVector<TString>> GetParameters(const TString&) const override {
ythrow TExternalSourceException() << "Only external table supports parameters";
}
+
+ virtual void ValidateProperties(const TString& properties) const override {
+ NKikimrSchemeOp::TExternalDataSourceProperties proto;
+ if (!proto.ParseFromString(properties)) {
+ ythrow TExternalSourceException() << "Internal error. Couldn't parse protobuf with properties for external data source";
+ }
+
+ for (const auto& [key, value]: proto.GetProperties()) {
+ if (AvailableProperties.contains(key)) {
+ continue;
+ }
+ ythrow TExternalSourceException() << "Unsupported property: " << key;
+ }
+ }
private:
const TString Name;
const TVector<TString> AuthMethods;
+ const TSet<TString> AvailableProperties;
};
}
-IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods) {
- return MakeIntrusive<TExternalDataSource>(name, authMethods);
+IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods, const TSet<TString>& availableProperties) {
+ return MakeIntrusive<TExternalDataSource>(name, authMethods, availableProperties);
}
}
diff --git a/ydb/core/external_sources/external_data_source.h b/ydb/core/external_sources/external_data_source.h
index 32034bfa1ec..46a154388f7 100644
--- a/ydb/core/external_sources/external_data_source.h
+++ b/ydb/core/external_sources/external_data_source.h
@@ -1,9 +1,10 @@
#pragma once
#include "external_source.h"
+#include <util/generic/set.h>
namespace NKikimr::NExternalSource {
-IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods);
+IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods, const TSet<TString>& availableProperties);
}
diff --git a/ydb/core/external_sources/external_source.h b/ydb/core/external_sources/external_source.h
index 4b8a05e18a8..d1ec3325992 100644
--- a/ydb/core/external_sources/external_source.h
+++ b/ydb/core/external_sources/external_source.h
@@ -48,6 +48,12 @@ struct IExternalSource : public TThrRefBase {
Can throw an exception in case of an error
*/
virtual TMap<TString, TVector<TString>> GetParameters(const TString& content) const = 0;
+
+ /*
+ Validation of external data source properties.
+ If an error occurs, an exception is thrown.
+ */
+ virtual void ValidateProperties(const TString& properties) const = 0;
};
}
diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp
index 78ff46f2778..074f0490052 100644
--- a/ydb/core/external_sources/external_source_factory.cpp
+++ b/ydb/core/external_sources/external_source_factory.cpp
@@ -33,8 +33,8 @@ private:
IExternalSourceFactory::TPtr CreateExternalSourceFactory() {
return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{
{"ObjectStorage", CreateObjectStorageExternalSource()},
- {"ClickHouse", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"})},
- {"PostgreSQL", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"})}
+ {"ClickHouse", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})},
+ {"PostgreSQL", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})}
});
}
diff --git a/ydb/core/external_sources/object_storage.cpp b/ydb/core/external_sources/object_storage.cpp
index 4029c4753e5..1ed10637e4e 100644
--- a/ydb/core/external_sources/object_storage.cpp
+++ b/ydb/core/external_sources/object_storage.cpp
@@ -1,6 +1,7 @@
#include "external_source.h"
#include <ydb/core/protos/external_sources.pb.h>
+#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.h>
#include <ydb/public/api/protos/ydb_status_codes.pb.h>
@@ -94,6 +95,17 @@ struct TObjectStorageExternalSource : public IExternalSource {
return parameters;
}
+ virtual void ValidateProperties(const TString& properties) const override {
+ NKikimrSchemeOp::TExternalDataSourceProperties proto;
+ if (!proto.ParseFromString(properties)) {
+ ythrow TExternalSourceException() << "Internal error. Couldn't parse protobuf with properties for external data source";
+ }
+
+ if (!proto.GetProperties().empty()) {
+ ythrow TExternalSourceException() << "ObjectStorage source doesn't support any properties";
+ }
+ }
+
private:
static NYql::TIssues Validate(const NKikimrExternalSources::TSchema& schema, const NKikimrExternalSources::TObjectStorage& objectStorage) {
NYql::TIssues issues;
diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
index c852f596f50..384a0e77b5e 100644
--- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
+++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp
@@ -50,6 +50,23 @@ void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescri
} else {
ythrow yexception() << "Internal error. Unknown auth method: " << authMethod;
}
+
+ static const TSet<TString> properties {
+ "database_name",
+ "protocol",
+ "mdb_cluster_id",
+ "use_tls"
+ };
+
+ for (const auto& property: properties) {
+ if (auto value = settings.GetFeaturesExtractor().Extract(property)) {
+ externaDataSourceDesc.MutableProperties()->MutableProperties()->insert({property, *value});
+ }
+ }
+
+ if (!settings.GetFeaturesExtractor().IsFinished()) {
+ ythrow yexception() << "Unknown property: " << settings.GetFeaturesExtractor().GetRemainedParamsString();
+ }
}
NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> SendSchemeRequest(TEvTxUserProxy::TEvProposeTransaction* request, TActorSystem* actorSystem, bool failedOnAlreadyExists = false)
diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp
index 9ba218f6269..aaef585dafe 100644
--- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp
+++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp
@@ -271,6 +271,7 @@ TTableMetadataResult GetExternalDataSourceMetadataResult(const NSchemeCache::TSc
tableMeta->ExternalSource.DataSourceLocation = description.GetLocation();
tableMeta->ExternalSource.DataSourceInstallation = description.GetInstallation();
tableMeta->ExternalSource.DataSourceAuth = description.GetAuth();
+ tableMeta->ExternalSource.Properties = description.GetProperties();
tableMeta->ExternalSource.DataSourcePath = tableName;
return result;
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
index fb93b5ff730..3ac2ec2431d 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
@@ -248,9 +248,12 @@ public:
THashMap<TString, TString> properties = {{
{"location", metadata.ExternalSource.DataSourceLocation },
- {"installation", metadata.ExternalSource.DataSourceInstallation }
+ {"installation", metadata.ExternalSource.DataSourceInstallation },
+ {"source_type", metadata.ExternalSource.Type}
}};
+ properties.insert(metadata.ExternalSource.Properties.GetProperties().begin(), metadata.ExternalSource.Properties.GetProperties().end());
+
switch (metadata.ExternalSource.DataSourceAuth.identity_case()) {
case NKikimrSchemeOp::TAuth::kServiceAccount:
properties["authMethod"] = "SERVICE_ACCOUNT";
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h
index 37f01919008..8af5e497ca9 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway.h
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h
@@ -373,6 +373,7 @@ struct TExternalSource {
TString AwsAccessKeyId;
TString AwsSecretAccessKey;
NKikimrSchemeOp::TAuth DataSourceAuth;
+ NKikimrSchemeOp::TExternalDataSourceProperties Properties;
};
struct TKikimrTableMetadata : public TThrRefBase {
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
index d53eac6e60f..22925a82ae7 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
@@ -447,6 +447,52 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) {
UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.AwsAccessKeyId, awsAccessKeyIdSecretValue);
UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.AwsSecretAccessKey, awsSecretAccessKeySecretValue);
}
+
+ Y_UNIT_TEST(TestLoadDataSourceProperties) {
+ TKikimrRunner kikimr;
+ kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ TString secretPasswordId = "myPasswordSecretId";
+ TString secretPasswordValue = "pswd";
+ CreateSecretObject(secretPasswordId, secretPasswordValue, session);
+
+ TString secretSaId = "mySa";
+ TString secretSaValue = "sign(mySa)";
+ CreateSecretObject(secretSaId, secretSaValue, session);
+
+ TString externalDataSourceName = "/Root/ExternalDataSource";
+ auto query = TStringBuilder() << R"(
+ CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH (
+ SOURCE_TYPE="PostgreSQL",
+ LOCATION="my-bucket",
+ AUTH_METHOD="MDB_BASIC",
+ SERVICE_ACCOUNT_ID="mysa",
+ SERVICE_ACCOUNT_SECRET_NAME=")" << secretSaId << R"(",
+ LOGIN="mylogin",
+ PASSWORD_SECRET_NAME=")" << secretPasswordId << R"(",
+ MDB_CLUSTER_ID="my_id",
+ DATABASE_NAME="my_db",
+ PROTOCOL="native",
+ USE_TLS="true"
+ );)";
+ auto result = session.ExecuteSchemeQuery(query).GetValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString());
+
+ auto responseFuture = GetIcGateway(kikimr.GetTestServer())->LoadTableMetadata(TestCluster, externalDataSourceName, IKikimrGateway::TLoadTableMetadataSettings());
+ responseFuture.Wait();
+
+ auto response = responseFuture.GetValue();
+ UNIT_ASSERT_C(response.Success(), response.Issues().ToOneLineString());
+ UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Password, secretPasswordValue);
+ UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.ServiceAccountIdSignature, secretSaValue);
+ UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().size(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("mdb_cluster_id"), "my_id");
+ UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("database_name"), "my_db");
+ UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("protocol"), "native");
+ UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("use_tls"), "true");
+ }
}
} // namespace NYql
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index e534803dc90..1be4e5efd4d 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -1781,6 +1781,10 @@ message TExternalTableReferences {
repeated TReference References = 1;
}
+message TExternalDataSourceProperties {
+ map<string, string> Properties = 1;
+}
+
message TExternalDataSourceDescription {
optional string Name = 1;
optional NKikimrProto.TPathID PathId = 2;
@@ -1789,4 +1793,5 @@ message TExternalDataSourceDescription {
optional string Location = 5;
optional string Installation = 6;
optional TAuth Auth = 7;
+ optional TExternalDataSourceProperties Properties = 8;
}
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
index 9bfdded52c2..1f091e46bf2 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -1818,7 +1818,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
externalDataSource->Installation = rowset.GetValue<Schema::ExternalDataSource::Installation>();
Y_PROTOBUF_SUPPRESS_NODISCARD externalDataSource->Auth.ParseFromString(rowset.GetValue<Schema::ExternalDataSource::Auth>());
Y_PROTOBUF_SUPPRESS_NODISCARD externalDataSource->ExternalTableReferences.ParseFromString(rowset.GetValue<Schema::ExternalDataSource::ExternalTableReferences>());
-
+ Y_PROTOBUF_SUPPRESS_NODISCARD externalDataSource->Properties.ParseFromString(rowset.GetValue<Schema::ExternalDataSource::Properties>());
Self->IncrementPathDbRefCount(pathId);
if (!rowset.Next())
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp
index 1f13b936c0e..c70d99bd8a4 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp
@@ -40,6 +40,14 @@ bool CheckAuth(const TString& authMethod, const TVector<TString>& availableAuthM
return true;
}
+bool ValidateProperties(const NKikimrSchemeOp::TExternalDataSourceProperties& properties, TString& errStr) {
+ if (properties.ByteSizeLong() > MAX_PROTOBUF_SIZE) {
+ errStr = Sprintf("Maximum size of properties must be less or equal equal to %u but got %lu", MAX_PROTOBUF_SIZE, properties.ByteSizeLong());
+ return false;
+ }
+ return true;
+}
+
bool ValidateAuth(const NKikimrSchemeOp::TAuth& auth, const NKikimr::NExternalSource::IExternalSource::TPtr& source, TString& errStr) {
if (auth.ByteSizeLong() > MAX_PROTOBUF_SIZE) {
errStr = Sprintf("Maximum size of authorization information must be less or equal equal to %u but got %lu", MAX_PROTOBUF_SIZE, auth.ByteSizeLong());
@@ -68,8 +76,10 @@ bool ValidateAuth(const NKikimrSchemeOp::TAuth& auth, const NKikimr::NExternalSo
bool Validate(const NKikimrSchemeOp::TExternalDataSourceDescription& desc, const NKikimr::NExternalSource::IExternalSourceFactory::TPtr& factory, TString& errStr) {
try {
auto source = factory->GetOrCreate(desc.GetSourceType());
+ source->ValidateProperties(desc.GetProperties().SerializeAsString());
return ValidateLocationAndInstallation(desc.GetLocation(), desc.GetInstallation(), errStr)
- && ValidateAuth(desc.GetAuth(), source, errStr);
+ && ValidateAuth(desc.GetAuth(), source, errStr)
+ && ValidateProperties(desc.GetProperties(), errStr);
} catch (...) {
errStr = CurrentExceptionMessage();
return false;
@@ -86,6 +96,7 @@ TExternalDataSourceInfo::TPtr CreateExternalDataSource(const NKikimrSchemeOp::TE
externalDataSoureInfo->Installation = desc.GetInstallation();
externalDataSoureInfo->AlterVersion = 1;
externalDataSoureInfo->Auth.CopyFrom(desc.GetAuth());
+ externalDataSoureInfo->Properties.CopyFrom(desc.GetProperties());
return externalDataSoureInfo;
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
index 65316279987..3cb255076b3 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -2675,7 +2675,8 @@ void TSchemeShard::PersistExternalDataSource(NIceDb::TNiceDb &db, TPathId pathId
NIceDb::TUpdate<Schema::ExternalDataSource::Location>{externalDataSourceInfo->Location},
NIceDb::TUpdate<Schema::ExternalDataSource::Installation>{externalDataSourceInfo->Installation},
NIceDb::TUpdate<Schema::ExternalDataSource::Auth>{externalDataSourceInfo->Auth.SerializeAsString()},
- NIceDb::TUpdate<Schema::ExternalDataSource::ExternalTableReferences>{externalDataSourceInfo->ExternalTableReferences.SerializeAsString()}
+ NIceDb::TUpdate<Schema::ExternalDataSource::ExternalTableReferences>{externalDataSourceInfo->ExternalTableReferences.SerializeAsString()},
+ NIceDb::TUpdate<Schema::ExternalDataSource::Properties>{externalDataSourceInfo->Properties.SerializeAsString()}
);
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h
index 61062967b04..7e9680a6f85 100644
--- a/ydb/core/tx/schemeshard/schemeshard_info_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h
@@ -3053,6 +3053,7 @@ struct TExternalDataSourceInfo: TSimpleRefCount<TExternalDataSourceInfo> {
TString Installation;
NKikimrSchemeOp::TAuth Auth;
NKikimrSchemeOp::TExternalTableReferences ExternalTableReferences;
+ NKikimrSchemeOp::TExternalDataSourceProperties Properties;
};
bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl,
diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
index 0125a207b12..0c58a0e3be1 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp
@@ -848,6 +848,7 @@ void TPathDescriber::DescribeExternalDataSource(const TActorContext&, TPathId pa
entry->SetLocation(externalDataSourceInfo->Location);
entry->SetInstallation(externalDataSourceInfo->Installation);
entry->MutableAuth()->CopyFrom(externalDataSourceInfo->Auth);
+ entry->MutableProperties()->CopyFrom(externalDataSourceInfo->Properties);
}
static bool ConsiderAsDropped(const TPath& path) {
diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h
index 0fd9a999f1c..1a2f6e1e5e3 100644
--- a/ydb/core/tx/schemeshard/schemeshard_schema.h
+++ b/ydb/core/tx/schemeshard/schemeshard_schema.h
@@ -1680,9 +1680,10 @@ struct Schema : NIceDb::Schema {
struct Installation : Column<6, NScheme::NTypeIds::Utf8> {};
struct Auth : Column<7, NScheme::NTypeIds::String> {};
struct ExternalTableReferences : Column<8, NScheme::NTypeIds::String> {};
+ struct Properties : Column<9, NScheme::NTypeIds::String> {};
using TKey = TableKey<OwnerPathId, LocalPathId>;
- using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, SourceType, Location, Installation, Auth, ExternalTableReferences>;
+ using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, SourceType, Location, Installation, Auth, ExternalTableReferences, Properties>;
};
struct PersQueueGroupStats : Table<106> {
diff --git a/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp b/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp
index bef999b7fa1..b9bcabbbec3 100644
--- a/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp
+++ b/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp
@@ -26,6 +26,34 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) {
TestLs(runtime, "/MyRoot/MyExternalDataSource", false, NLs::PathExist);
}
+ Y_UNIT_TEST(CreateExternalDataSourceWithProperties) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ TestCreateExternalDataSource(runtime, txId++, "/MyRoot",R"(
+ Name: "MyExternalDataSource"
+ SourceType: "PostgreSQL"
+ Location: "localhost:5432"
+ Auth {
+ Basic {
+ Login: "my_login",
+ PasswordSecretName: "password_secret"
+ }
+ }
+ Properties {
+ Properties {
+ key: "mdb_cluster_id",
+ value: "id"
+ }
+ }
+ )", {NKikimrScheme::StatusAccepted});
+
+ env.TestWaitNotification(runtime, 100);
+
+ TestLs(runtime, "/MyRoot/MyExternalDataSource", false, NLs::PathExist);
+ }
+
Y_UNIT_TEST(DropExternalDataSource) {
TTestBasicRuntime runtime;
TTestEnv env(runtime);
diff --git a/ydb/library/yql/sql/v1/sql_translation.cpp b/ydb/library/yql/sql/v1/sql_translation.cpp
index 90e32fc1ff7..024037cc7bd 100644
--- a/ydb/library/yql/sql/v1/sql_translation.cpp
+++ b/ydb/library/yql/sql/v1/sql_translation.cpp
@@ -4197,16 +4197,10 @@ bool TSqlTranslation::StoreDataSourceSettingsEntry(const TIdentifier& id, const
return false;
}
- if (IsIn({"source_type", "installation", "location",
- "auth_method", "service_account_id", "service_account_secret_name",
- "login", "password_secret_name", "aws_access_key_id_secret_name", "aws_secret_access_key_secret_name"}, key)) {
- if (!StoreString(*value, result[key], Ctx, to_upper(key))) {
- return false;
- }
- } else {
- Ctx.Error() << "Unknown external data source setting: " << id.Name;
+ if (!StoreString(*value, result[key], Ctx, to_upper(key))) {
return false;
}
+
return true;
}
@@ -4229,10 +4223,6 @@ bool TSqlTranslation::ParseExternalDataSourceSettings(std::map<TString, TDeferre
if (!ValidateAuthMethod(result)) {
return false;
}
- if (result.find("installation") == result.end() && result.find("location") == result.end()) {
- Ctx.Error() << "INSTALLATION or LOCATION must be specified";
- return false;
- }
return true;
}
diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp
index 80985f4dfcc..64d816f6827 100644
--- a/ydb/library/yql/sql/v1/sql_ut.cpp
+++ b/ydb/library/yql/sql/v1/sql_ut.cpp
@@ -5416,27 +5416,10 @@ Y_UNIT_TEST_SUITE(ExternalDataSource) {
USE plato;
CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
SOURCE_TYPE="ObjectStorage",
- AUTH_METHOD="NONE"
- );
- )" , "<main>:5:33: Error: INSTALLATION or LOCATION must be specified\n");
-
- ExpectFailWithError(R"(
- USE plato;
- CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
- SOURCE_TYPE="ObjectStorage",
LOCATION="my-bucket"
);
)" , "<main>:5:30: Error: AUTH_METHOD requires key\n");
- ExpectFailWithError(R"(
- USE plato;
- CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
- SOURCE_TYPE="ObjectStorage",
- LOCATION="my-bucket",
- AUTH_METHOD="NONE",
- OTHER="VALUE"
- );
- )" , "<main>:7:21: Error: Unknown external data source setting: OTHER\n");
ExpectFailWithError(R"(
USE plato;
diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
index 8e408d95736..17db38d82b0 100644
--- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
+++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
@@ -6962,6 +6962,11 @@
"ColumnId": 8,
"ColumnName": "ExternalTableReferences",
"ColumnType": "String"
+ },
+ {
+ "ColumnId": 9,
+ "ColumnName": "Properties",
+ "ColumnType": "String"
}
],
"ColumnsDropped": [],
@@ -6975,7 +6980,8 @@
5,
6,
7,
- 8
+ 8,
+ 9
],
"RoomID": 0,
"Codec": 0,