diff options
author | hcpp <[email protected]> | 2023-09-01 13:46:24 +0300 |
---|---|---|
committer | hcpp <[email protected]> | 2023-09-01 14:02:27 +0300 |
commit | bf418335042882be6345a85cce0fa92e8c95537a (patch) | |
tree | bb3567b5bfd60b086f414a218e71df52d9e66d39 | |
parent | b7bb944233f923121a9fc3a7cdeaaaad25a9d6df (diff) |
new data source properties have been added
21 files changed, 172 insertions, 41 deletions
diff --git a/ydb/core/external_sources/external_data_source.cpp b/ydb/core/external_sources/external_data_source.cpp index f0e11708535..e35a2643fa6 100644 --- a/ydb/core/external_sources/external_data_source.cpp +++ b/ydb/core/external_sources/external_data_source.cpp @@ -1,13 +1,15 @@ #include "external_data_source.h" +#include <ydb/core/protos/flat_scheme_op.pb.h> namespace NKikimr::NExternalSource { namespace { struct TExternalDataSource : public IExternalSource { - TExternalDataSource(const TString& name, const TVector<TString>& authMethods) + TExternalDataSource(const TString& name, const TVector<TString>& authMethods, const TSet<TString>& availableProperties) : Name(name) , AuthMethods(authMethods) + , AvailableProperties(availableProperties) {} virtual TString Pack(const NKikimrExternalSources::TSchema&, @@ -30,15 +32,30 @@ struct TExternalDataSource : public IExternalSource { virtual TMap<TString, TVector<TString>> GetParameters(const TString&) const override { ythrow TExternalSourceException() << "Only external table supports parameters"; } + + virtual void ValidateProperties(const TString& properties) const override { + NKikimrSchemeOp::TExternalDataSourceProperties proto; + if (!proto.ParseFromString(properties)) { + ythrow TExternalSourceException() << "Internal error. Couldn't parse protobuf with properties for external data source"; + } + + for (const auto& [key, value]: proto.GetProperties()) { + if (AvailableProperties.contains(key)) { + continue; + } + ythrow TExternalSourceException() << "Unsupported property: " << key; + } + } private: const TString Name; const TVector<TString> AuthMethods; + const TSet<TString> AvailableProperties; }; } -IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods) { - return MakeIntrusive<TExternalDataSource>(name, authMethods); +IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods, const TSet<TString>& availableProperties) { + return MakeIntrusive<TExternalDataSource>(name, authMethods, availableProperties); } } diff --git a/ydb/core/external_sources/external_data_source.h b/ydb/core/external_sources/external_data_source.h index 32034bfa1ec..46a154388f7 100644 --- a/ydb/core/external_sources/external_data_source.h +++ b/ydb/core/external_sources/external_data_source.h @@ -1,9 +1,10 @@ #pragma once #include "external_source.h" +#include <util/generic/set.h> namespace NKikimr::NExternalSource { -IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods); +IExternalSource::TPtr CreateExternalDataSource(const TString& name, const TVector<TString>& authMethods, const TSet<TString>& availableProperties); } diff --git a/ydb/core/external_sources/external_source.h b/ydb/core/external_sources/external_source.h index 4b8a05e18a8..d1ec3325992 100644 --- a/ydb/core/external_sources/external_source.h +++ b/ydb/core/external_sources/external_source.h @@ -48,6 +48,12 @@ struct IExternalSource : public TThrRefBase { Can throw an exception in case of an error */ virtual TMap<TString, TVector<TString>> GetParameters(const TString& content) const = 0; + + /* + Validation of external data source properties. + If an error occurs, an exception is thrown. + */ + virtual void ValidateProperties(const TString& properties) const = 0; }; } diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp index 78ff46f2778..074f0490052 100644 --- a/ydb/core/external_sources/external_source_factory.cpp +++ b/ydb/core/external_sources/external_source_factory.cpp @@ -33,8 +33,8 @@ private: IExternalSourceFactory::TPtr CreateExternalSourceFactory() { return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{ {"ObjectStorage", CreateObjectStorageExternalSource()}, - {"ClickHouse", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"})}, - {"PostgreSQL", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"})} + {"ClickHouse", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})}, + {"PostgreSQL", CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"})} }); } diff --git a/ydb/core/external_sources/object_storage.cpp b/ydb/core/external_sources/object_storage.cpp index 4029c4753e5..1ed10637e4e 100644 --- a/ydb/core/external_sources/object_storage.cpp +++ b/ydb/core/external_sources/object_storage.cpp @@ -1,6 +1,7 @@ #include "external_source.h" #include <ydb/core/protos/external_sources.pb.h> +#include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> #include <ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.h> #include <ydb/public/api/protos/ydb_status_codes.pb.h> @@ -94,6 +95,17 @@ struct TObjectStorageExternalSource : public IExternalSource { return parameters; } + virtual void ValidateProperties(const TString& properties) const override { + NKikimrSchemeOp::TExternalDataSourceProperties proto; + if (!proto.ParseFromString(properties)) { + ythrow TExternalSourceException() << "Internal error. Couldn't parse protobuf with properties for external data source"; + } + + if (!proto.GetProperties().empty()) { + ythrow TExternalSourceException() << "ObjectStorage source doesn't support any properties"; + } + } + private: static NYql::TIssues Validate(const NKikimrExternalSources::TSchema& schema, const NKikimrExternalSources::TObjectStorage& objectStorage) { NYql::TIssues issues; diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp index c852f596f50..384a0e77b5e 100644 --- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp @@ -50,6 +50,23 @@ void FillCreateExternalDataSourceDesc(NKikimrSchemeOp::TExternalDataSourceDescri } else { ythrow yexception() << "Internal error. Unknown auth method: " << authMethod; } + + static const TSet<TString> properties { + "database_name", + "protocol", + "mdb_cluster_id", + "use_tls" + }; + + for (const auto& property: properties) { + if (auto value = settings.GetFeaturesExtractor().Extract(property)) { + externaDataSourceDesc.MutableProperties()->MutableProperties()->insert({property, *value}); + } + } + + if (!settings.GetFeaturesExtractor().IsFinished()) { + ythrow yexception() << "Unknown property: " << settings.GetFeaturesExtractor().GetRemainedParamsString(); + } } NThreading::TFuture<TExternalDataSourceManager::TYqlConclusionStatus> SendSchemeRequest(TEvTxUserProxy::TEvProposeTransaction* request, TActorSystem* actorSystem, bool failedOnAlreadyExists = false) diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp index 9ba218f6269..aaef585dafe 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp @@ -271,6 +271,7 @@ TTableMetadataResult GetExternalDataSourceMetadataResult(const NSchemeCache::TSc tableMeta->ExternalSource.DataSourceLocation = description.GetLocation(); tableMeta->ExternalSource.DataSourceInstallation = description.GetInstallation(); tableMeta->ExternalSource.DataSourceAuth = description.GetAuth(); + tableMeta->ExternalSource.Properties = description.GetProperties(); tableMeta->ExternalSource.DataSourcePath = tableName; return result; } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index fb93b5ff730..3ac2ec2431d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -248,9 +248,12 @@ public: THashMap<TString, TString> properties = {{ {"location", metadata.ExternalSource.DataSourceLocation }, - {"installation", metadata.ExternalSource.DataSourceInstallation } + {"installation", metadata.ExternalSource.DataSourceInstallation }, + {"source_type", metadata.ExternalSource.Type} }}; + properties.insert(metadata.ExternalSource.Properties.GetProperties().begin(), metadata.ExternalSource.Properties.GetProperties().end()); + switch (metadata.ExternalSource.DataSourceAuth.identity_case()) { case NKikimrSchemeOp::TAuth::kServiceAccount: properties["authMethod"] = "SERVICE_ACCOUNT"; diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 37f01919008..8af5e497ca9 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -373,6 +373,7 @@ struct TExternalSource { TString AwsAccessKeyId; TString AwsSecretAccessKey; NKikimrSchemeOp::TAuth DataSourceAuth; + NKikimrSchemeOp::TExternalDataSourceProperties Properties; }; struct TKikimrTableMetadata : public TThrRefBase { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index d53eac6e60f..22925a82ae7 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -447,6 +447,52 @@ Y_UNIT_TEST_SUITE(KikimrIcGateway) { UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.AwsAccessKeyId, awsAccessKeyIdSecretValue); UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.AwsSecretAccessKey, awsSecretAccessKeySecretValue); } + + Y_UNIT_TEST(TestLoadDataSourceProperties) { + TKikimrRunner kikimr; + kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + TString secretPasswordId = "myPasswordSecretId"; + TString secretPasswordValue = "pswd"; + CreateSecretObject(secretPasswordId, secretPasswordValue, session); + + TString secretSaId = "mySa"; + TString secretSaValue = "sign(mySa)"; + CreateSecretObject(secretSaId, secretSaValue, session); + + TString externalDataSourceName = "/Root/ExternalDataSource"; + auto query = TStringBuilder() << R"( + CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH ( + SOURCE_TYPE="PostgreSQL", + LOCATION="my-bucket", + AUTH_METHOD="MDB_BASIC", + SERVICE_ACCOUNT_ID="mysa", + SERVICE_ACCOUNT_SECRET_NAME=")" << secretSaId << R"(", + LOGIN="mylogin", + PASSWORD_SECRET_NAME=")" << secretPasswordId << R"(", + MDB_CLUSTER_ID="my_id", + DATABASE_NAME="my_db", + PROTOCOL="native", + USE_TLS="true" + );)"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + + auto responseFuture = GetIcGateway(kikimr.GetTestServer())->LoadTableMetadata(TestCluster, externalDataSourceName, IKikimrGateway::TLoadTableMetadataSettings()); + responseFuture.Wait(); + + auto response = responseFuture.GetValue(); + UNIT_ASSERT_C(response.Success(), response.Issues().ToOneLineString()); + UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Password, secretPasswordValue); + UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.ServiceAccountIdSignature, secretSaValue); + UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().size(), 4); + UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("mdb_cluster_id"), "my_id"); + UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("database_name"), "my_db"); + UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("protocol"), "native"); + UNIT_ASSERT_VALUES_EQUAL(response.Metadata->ExternalSource.Properties.GetProperties().at("use_tls"), "true"); + } } } // namespace NYql diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index e534803dc90..1be4e5efd4d 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -1781,6 +1781,10 @@ message TExternalTableReferences { repeated TReference References = 1; } +message TExternalDataSourceProperties { + map<string, string> Properties = 1; +} + message TExternalDataSourceDescription { optional string Name = 1; optional NKikimrProto.TPathID PathId = 2; @@ -1789,4 +1793,5 @@ message TExternalDataSourceDescription { optional string Location = 5; optional string Installation = 6; optional TAuth Auth = 7; + optional TExternalDataSourceProperties Properties = 8; } diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 9bfdded52c2..1f091e46bf2 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -1818,7 +1818,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { externalDataSource->Installation = rowset.GetValue<Schema::ExternalDataSource::Installation>(); Y_PROTOBUF_SUPPRESS_NODISCARD externalDataSource->Auth.ParseFromString(rowset.GetValue<Schema::ExternalDataSource::Auth>()); Y_PROTOBUF_SUPPRESS_NODISCARD externalDataSource->ExternalTableReferences.ParseFromString(rowset.GetValue<Schema::ExternalDataSource::ExternalTableReferences>()); - + Y_PROTOBUF_SUPPRESS_NODISCARD externalDataSource->Properties.ParseFromString(rowset.GetValue<Schema::ExternalDataSource::Properties>()); Self->IncrementPathDbRefCount(pathId); if (!rowset.Next()) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp index 1f13b936c0e..c70d99bd8a4 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp @@ -40,6 +40,14 @@ bool CheckAuth(const TString& authMethod, const TVector<TString>& availableAuthM return true; } +bool ValidateProperties(const NKikimrSchemeOp::TExternalDataSourceProperties& properties, TString& errStr) { + if (properties.ByteSizeLong() > MAX_PROTOBUF_SIZE) { + errStr = Sprintf("Maximum size of properties must be less or equal equal to %u but got %lu", MAX_PROTOBUF_SIZE, properties.ByteSizeLong()); + return false; + } + return true; +} + bool ValidateAuth(const NKikimrSchemeOp::TAuth& auth, const NKikimr::NExternalSource::IExternalSource::TPtr& source, TString& errStr) { if (auth.ByteSizeLong() > MAX_PROTOBUF_SIZE) { errStr = Sprintf("Maximum size of authorization information must be less or equal equal to %u but got %lu", MAX_PROTOBUF_SIZE, auth.ByteSizeLong()); @@ -68,8 +76,10 @@ bool ValidateAuth(const NKikimrSchemeOp::TAuth& auth, const NKikimr::NExternalSo bool Validate(const NKikimrSchemeOp::TExternalDataSourceDescription& desc, const NKikimr::NExternalSource::IExternalSourceFactory::TPtr& factory, TString& errStr) { try { auto source = factory->GetOrCreate(desc.GetSourceType()); + source->ValidateProperties(desc.GetProperties().SerializeAsString()); return ValidateLocationAndInstallation(desc.GetLocation(), desc.GetInstallation(), errStr) - && ValidateAuth(desc.GetAuth(), source, errStr); + && ValidateAuth(desc.GetAuth(), source, errStr) + && ValidateProperties(desc.GetProperties(), errStr); } catch (...) { errStr = CurrentExceptionMessage(); return false; @@ -86,6 +96,7 @@ TExternalDataSourceInfo::TPtr CreateExternalDataSource(const NKikimrSchemeOp::TE externalDataSoureInfo->Installation = desc.GetInstallation(); externalDataSoureInfo->AlterVersion = 1; externalDataSoureInfo->Auth.CopyFrom(desc.GetAuth()); + externalDataSoureInfo->Properties.CopyFrom(desc.GetProperties()); return externalDataSoureInfo; } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 65316279987..3cb255076b3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -2675,7 +2675,8 @@ void TSchemeShard::PersistExternalDataSource(NIceDb::TNiceDb &db, TPathId pathId NIceDb::TUpdate<Schema::ExternalDataSource::Location>{externalDataSourceInfo->Location}, NIceDb::TUpdate<Schema::ExternalDataSource::Installation>{externalDataSourceInfo->Installation}, NIceDb::TUpdate<Schema::ExternalDataSource::Auth>{externalDataSourceInfo->Auth.SerializeAsString()}, - NIceDb::TUpdate<Schema::ExternalDataSource::ExternalTableReferences>{externalDataSourceInfo->ExternalTableReferences.SerializeAsString()} + NIceDb::TUpdate<Schema::ExternalDataSource::ExternalTableReferences>{externalDataSourceInfo->ExternalTableReferences.SerializeAsString()}, + NIceDb::TUpdate<Schema::ExternalDataSource::Properties>{externalDataSourceInfo->Properties.SerializeAsString()} ); } diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 61062967b04..7e9680a6f85 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3053,6 +3053,7 @@ struct TExternalDataSourceInfo: TSimpleRefCount<TExternalDataSourceInfo> { TString Installation; NKikimrSchemeOp::TAuth Auth; NKikimrSchemeOp::TExternalTableReferences ExternalTableReferences; + NKikimrSchemeOp::TExternalDataSourceProperties Properties; }; bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl, diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 0125a207b12..0c58a0e3be1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -848,6 +848,7 @@ void TPathDescriber::DescribeExternalDataSource(const TActorContext&, TPathId pa entry->SetLocation(externalDataSourceInfo->Location); entry->SetInstallation(externalDataSourceInfo->Installation); entry->MutableAuth()->CopyFrom(externalDataSourceInfo->Auth); + entry->MutableProperties()->CopyFrom(externalDataSourceInfo->Properties); } static bool ConsiderAsDropped(const TPath& path) { diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 0fd9a999f1c..1a2f6e1e5e3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1680,9 +1680,10 @@ struct Schema : NIceDb::Schema { struct Installation : Column<6, NScheme::NTypeIds::Utf8> {}; struct Auth : Column<7, NScheme::NTypeIds::String> {}; struct ExternalTableReferences : Column<8, NScheme::NTypeIds::String> {}; + struct Properties : Column<9, NScheme::NTypeIds::String> {}; using TKey = TableKey<OwnerPathId, LocalPathId>; - using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, SourceType, Location, Installation, Auth, ExternalTableReferences>; + using TColumns = TableColumns<OwnerPathId, LocalPathId, AlterVersion, SourceType, Location, Installation, Auth, ExternalTableReferences, Properties>; }; struct PersQueueGroupStats : Table<106> { diff --git a/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp b/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp index bef999b7fa1..b9bcabbbec3 100644 --- a/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp +++ b/ydb/core/tx/schemeshard/ut_external_data_source/ut_external_data_source.cpp @@ -26,6 +26,34 @@ Y_UNIT_TEST_SUITE(TExternalDataSourceTest) { TestLs(runtime, "/MyRoot/MyExternalDataSource", false, NLs::PathExist); } + Y_UNIT_TEST(CreateExternalDataSourceWithProperties) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateExternalDataSource(runtime, txId++, "/MyRoot",R"( + Name: "MyExternalDataSource" + SourceType: "PostgreSQL" + Location: "localhost:5432" + Auth { + Basic { + Login: "my_login", + PasswordSecretName: "password_secret" + } + } + Properties { + Properties { + key: "mdb_cluster_id", + value: "id" + } + } + )", {NKikimrScheme::StatusAccepted}); + + env.TestWaitNotification(runtime, 100); + + TestLs(runtime, "/MyRoot/MyExternalDataSource", false, NLs::PathExist); + } + Y_UNIT_TEST(DropExternalDataSource) { TTestBasicRuntime runtime; TTestEnv env(runtime); diff --git a/ydb/library/yql/sql/v1/sql_translation.cpp b/ydb/library/yql/sql/v1/sql_translation.cpp index 90e32fc1ff7..024037cc7bd 100644 --- a/ydb/library/yql/sql/v1/sql_translation.cpp +++ b/ydb/library/yql/sql/v1/sql_translation.cpp @@ -4197,16 +4197,10 @@ bool TSqlTranslation::StoreDataSourceSettingsEntry(const TIdentifier& id, const return false; } - if (IsIn({"source_type", "installation", "location", - "auth_method", "service_account_id", "service_account_secret_name", - "login", "password_secret_name", "aws_access_key_id_secret_name", "aws_secret_access_key_secret_name"}, key)) { - if (!StoreString(*value, result[key], Ctx, to_upper(key))) { - return false; - } - } else { - Ctx.Error() << "Unknown external data source setting: " << id.Name; + if (!StoreString(*value, result[key], Ctx, to_upper(key))) { return false; } + return true; } @@ -4229,10 +4223,6 @@ bool TSqlTranslation::ParseExternalDataSourceSettings(std::map<TString, TDeferre if (!ValidateAuthMethod(result)) { return false; } - if (result.find("installation") == result.end() && result.find("location") == result.end()) { - Ctx.Error() << "INSTALLATION or LOCATION must be specified"; - return false; - } return true; } diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 80985f4dfcc..64d816f6827 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -5416,27 +5416,10 @@ Y_UNIT_TEST_SUITE(ExternalDataSource) { USE plato; CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( SOURCE_TYPE="ObjectStorage", - AUTH_METHOD="NONE" - ); - )" , "<main>:5:33: Error: INSTALLATION or LOCATION must be specified\n"); - - ExpectFailWithError(R"( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", LOCATION="my-bucket" ); )" , "<main>:5:30: Error: AUTH_METHOD requires key\n"); - ExpectFailWithError(R"( - USE plato; - CREATE EXTERNAL DATA SOURCE MyDataSource WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="NONE", - OTHER="VALUE" - ); - )" , "<main>:7:21: Error: Unknown external data source setting: OTHER\n"); ExpectFailWithError(R"( USE plato; diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 8e408d95736..17db38d82b0 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -6962,6 +6962,11 @@ "ColumnId": 8, "ColumnName": "ExternalTableReferences", "ColumnType": "String" + }, + { + "ColumnId": 9, + "ColumnName": "Properties", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -6975,7 +6980,8 @@ 5, 6, 7, - 8 + 8, + 9 ], "RoomID": 0, "Codec": 0, |