diff options
| author | Vitaly Isaev <vitalyisaev@ydb.tech> | 2024-12-11 14:57:57 +0000 |
|---|---|---|
| committer | Vitaly Isaev <vitalyisaev@ydb.tech> | 2024-12-12 10:20:54 +0000 |
| commit | fe16216e477819967dd29d863927d6618194dabc (patch) | |
| tree | d33510c3dad5cec821965a8879a382fc205d7eb4 | |
| parent | 063b40415d853541d6c377d9f6d9b2186c92e528 (diff) | |
| download | ydb-fe16216e477819967dd29d863927d6618194dabc.tar.gz | |
YDB FQ: turning gateways_config.proto into a file without external dependencies
47 files changed, 225 insertions, 229 deletions
diff --git a/ydb/core/fq/libs/actors/clusters_from_connections.cpp b/ydb/core/fq/libs/actors/clusters_from_connections.cpp index 90434706f2..1249aeafb6 100644 --- a/ydb/core/fq/libs/actors/clusters_from_connections.cpp +++ b/ydb/core/fq/libs/actors/clusters_from_connections.cpp @@ -1,7 +1,7 @@ #include "clusters_from_connections.h" #include <yql/essentials/providers/common/provider/yql_provider_names.h> -#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> +#include <yql/essentials/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.h> #include <yql/essentials/utils/url_builder.h> #include <ydb/library/actors/http/http.h> @@ -112,7 +112,7 @@ void FillGenericClusterConfigBase( TGenericClusterConfig& clusterCfg, const TConnection& connection, const TString& connectionName, - NConnector::NApi::EDataSourceKind dataSourceKind, + NYql::EGenericDataSourceKind dataSourceKind, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures ) { @@ -128,21 +128,21 @@ void FillGenericClusterConfigBase( // In YQv1 we just hardcode the appropriate protocols here. // In YQv2 protocol can be configured via `CREATE EXTERNAL DATA SOURCE` params. switch (dataSourceKind) { - case NYql::NConnector::NApi::CLICKHOUSE: - clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::NConnector::NApi::EProtocol::NATIVE : NYql::NConnector::NApi::EProtocol::HTTP); + case NYql::EGenericDataSourceKind::CLICKHOUSE: + clusterCfg.SetProtocol(common.GetUseNativeProtocolForClickHouse() ? NYql::EGenericProtocol::NATIVE : NYql::EGenericProtocol::HTTP); break; - case NYql::NConnector::NApi::GREENPLUM: - clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE); + case NYql::EGenericDataSourceKind::GREENPLUM: + clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE); break; - case NYql::NConnector::NApi::MYSQL: - clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE); + case NYql::EGenericDataSourceKind::MYSQL: + clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE); break; - case NYql::NConnector::NApi::POSTGRESQL: - clusterCfg.SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE); + case NYql::EGenericDataSourceKind::POSTGRESQL: + clusterCfg.SetProtocol(NYql::EGenericProtocol::NATIVE); break; default: ythrow yexception() << "Unexpected data source kind: '" - << NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) << "'"; + << NYql::EGenericDataSourceKind_Name(dataSourceKind) << "'"; } ValidateGenericClusterConfig(clusterCfg, "NFq::FillGenericClusterFromConfig"); @@ -154,7 +154,7 @@ void FillGenericClusterConfig( TGenericClusterConfig& clusterCfg, const TConnection& connection, const TString& connectionName, - NConnector::NApi::EDataSourceKind dataSourceKind, + NYql::EGenericDataSourceKind dataSourceKind, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures ) { @@ -167,7 +167,7 @@ void FillGenericClusterConfig<FederatedQuery::PostgreSQLCluster>( TGenericClusterConfig& clusterCfg, const FederatedQuery::PostgreSQLCluster& connection, const TString& connectionName, - NConnector::NApi::EDataSourceKind dataSourceKind, + NYql::EGenericDataSourceKind dataSourceKind, const TString& authToken, const THashMap<TString, TString>& accountIdSignatures ){ @@ -227,8 +227,8 @@ void AddClustersFromConnections( case FederatedQuery::ConnectionSetting::kYdbDatabase: { const auto& db = conn.content().setting().ydb_database(); auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping(); - clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::YDB); - clusterCfg->SetProtocol(NYql::NConnector::NApi::EProtocol::NATIVE); + clusterCfg->SetKind(NYql::EGenericDataSourceKind::YDB); + clusterCfg->SetProtocol(NYql::EGenericProtocol::NATIVE); clusterCfg->SetName(connectionName); clusterCfg->SetDatabaseId(db.database_id()); clusterCfg->SetUseSsl(!common.GetDisableSslForGenericDataSources()); @@ -242,7 +242,7 @@ void AddClustersFromConnections( *gatewaysConfig.MutableGeneric()->AddClusterMapping(), conn.content().setting().clickhouse_cluster(), connectionName, - NYql::NConnector::NApi::EDataSourceKind::CLICKHOUSE, + NYql::EGenericDataSourceKind::CLICKHOUSE, authToken, accountIdSignatures); clusters.emplace(connectionName, GenericProviderName); @@ -275,7 +275,7 @@ void AddClustersFromConnections( *gatewaysConfig.MutableGeneric()->AddClusterMapping(), conn.content().setting().postgresql_cluster(), connectionName, - NYql::NConnector::NApi::EDataSourceKind::POSTGRESQL, + NYql::EGenericDataSourceKind::POSTGRESQL, authToken, accountIdSignatures); clusters.emplace(connectionName, GenericProviderName); @@ -287,7 +287,7 @@ void AddClustersFromConnections( *gatewaysConfig.MutableGeneric()->AddClusterMapping(), conn.content().setting().greenplum_cluster(), connectionName, - NYql::NConnector::NApi::EDataSourceKind::GREENPLUM, + NYql::EGenericDataSourceKind::GREENPLUM, authToken, accountIdSignatures); clusters.emplace(connectionName, GenericProviderName); @@ -299,7 +299,7 @@ void AddClustersFromConnections( *gatewaysConfig.MutableGeneric()->AddClusterMapping(), conn.content().setting().mysql_cluster(), connectionName, - NYql::NConnector::NApi::EDataSourceKind::MYSQL, + NYql::EGenericDataSourceKind::MYSQL, authToken, accountIdSignatures); clusters.emplace(connectionName, GenericProviderName); @@ -308,7 +308,7 @@ void AddClustersFromConnections( case FederatedQuery::ConnectionSetting::kLogging: { const auto& connection = conn.content().setting().logging(); auto* clusterCfg = gatewaysConfig.MutableGeneric()->AddClusterMapping(); - clusterCfg->SetKind(NYql::NConnector::NApi::EDataSourceKind::LOGGING); + clusterCfg->SetKind(NYql::EGenericDataSourceKind::LOGGING); clusterCfg->SetName(connectionName); clusterCfg->mutable_datasourceoptions()->insert({"folder_id", connection.folder_id()}); FillClusterAuth(*clusterCfg, connection.auth(), authToken, accountIdSignatures); diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp index 98be09d733..ef033e7047 100644 --- a/ydb/core/fq/libs/actors/database_resolver.cpp +++ b/ydb/core/fq/libs/actors/database_resolver.cpp @@ -29,7 +29,7 @@ using TParser = std::function<TDatabaseDescription( NJson::TJsonValue& body, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls, - NConnector::NApi::EProtocol protocol + NYql::EGenericProtocol protocol )>; using TParsers = THashMap<NYql::EDatabaseType, TParser>; @@ -292,7 +292,7 @@ public: .SetErrorTtl(TDuration::Minutes(1)) .SetMaxSize(1000000)) { - auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NConnector::NApi::EProtocol) { + auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NYql::EGenericProtocol) { bool secure = false; TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust(); TString prefix("/?database="); @@ -333,7 +333,7 @@ public: NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls, - NConnector::NApi::EProtocol protocol) + NYql::EGenericProtocol protocol) { auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol); // TODO: Take explicit field from MVP @@ -349,7 +349,7 @@ public: NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls, - NConnector::NApi::EProtocol protocol + NYql::EGenericProtocol protocol ) { NYql::IMdbEndpointGenerator::TEndpoint endpoint; TVector<TString> aliveHosts; @@ -380,7 +380,7 @@ public: NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls, - NConnector::NApi::EProtocol protocol + NYql::EGenericProtocol protocol ) { NYql::IMdbEndpointGenerator::TEndpoint endpoint; TVector<TString> aliveHosts; @@ -427,7 +427,7 @@ public: NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls, - NConnector::NApi::EProtocol protocol + NYql::EGenericProtocol protocol ) { NYql::IMdbEndpointGenerator::TEndpoint endpoint; TString aliveHost; @@ -465,7 +465,7 @@ public: NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls, - NConnector::NApi::EProtocol protocol + NYql::EGenericProtocol protocol ) { NYql::IMdbEndpointGenerator::TEndpoint endpoint; TVector<TString> aliveHosts; diff --git a/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp b/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp index 7295d7beb8..59a4cb47ae 100644 --- a/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp +++ b/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { void Test( NYql::EDatabaseType databaseType, - NYql::NConnector::NApi::EProtocol protocol, + NYql::EGenericProtocol protocol, const TString& getUrl, const TString& status, const TString& responseBody, @@ -186,7 +186,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(Ydb_Serverless) { Test( NYql::EDatabaseType::Ydb, - NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED, + NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED, "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh", "200", R"( @@ -218,7 +218,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Test( NYql::EDatabaseType::Ydb, - NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED, + NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED, "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh", "", "", @@ -237,7 +237,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(Ydb_Dedicated) { Test( NYql::EDatabaseType::Ydb, - NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED, + NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED, "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh", "200", R"( @@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(DataStreams_Serverless) { Test( NYql::EDatabaseType::DataStreams, - NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED, + NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED, "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh", "200", R"( @@ -280,7 +280,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(DataStreams_Dedicated) { Test( NYql::EDatabaseType::DataStreams, - NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED, + NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED, "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh", "200", R"( @@ -302,7 +302,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(ClickHouseNative) { Test( NYql::EDatabaseType::ClickHouse, - NYql::NConnector::NApi::EProtocol::NATIVE, + NYql::EGenericProtocol::NATIVE, "https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts", "200", R"({ @@ -336,7 +336,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(ClickHouseHttp) { Test( NYql::EDatabaseType::ClickHouse, - NYql::NConnector::NApi::EProtocol::HTTP, + NYql::EGenericProtocol::HTTP, "https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts", "200", R"({ @@ -381,7 +381,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Test( NYql::EDatabaseType::ClickHouse, - NYql::NConnector::NApi::EProtocol::HTTP, + NYql::EGenericProtocol::HTTP, "https://mdb.api.cloud.yandex.net:443/managed-clickhouse/v1/clusters/etn021us5r9rhld1vgbh/hosts", "403", R"( @@ -405,7 +405,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(PostgreSQL) { Test( NYql::EDatabaseType::PostgreSQL, - NYql::NConnector::NApi::EProtocol::NATIVE, + NYql::EGenericProtocol::NATIVE, "https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts", "200", R"({ @@ -454,7 +454,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Test( NYql::EDatabaseType::PostgreSQL, - NYql::NConnector::NApi::EProtocol::NATIVE, + NYql::EGenericProtocol::NATIVE, "https://mdb.api.cloud.yandex.net:443/managed-postgresql/v1/clusters/etn021us5r9rhld1vgbh/hosts", "403", R"( @@ -478,7 +478,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(Greenplum_MasterNode) { Test( NYql::EDatabaseType::Greenplum, - NYql::NConnector::NApi::EProtocol::NATIVE, + NYql::EGenericProtocol::NATIVE, "https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts", "200", R"({ @@ -520,7 +520,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Test( NYql::EDatabaseType::Greenplum, - NYql::NConnector::NApi::EProtocol::NATIVE, + NYql::EGenericProtocol::NATIVE, "https://mdb.api.cloud.yandex.net:443/managed-greenplum/v1/clusters/etn021us5r9rhld1vgbh/master-hosts", "403", R"( @@ -542,7 +542,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Y_UNIT_TEST(MySQL) { Test( NYql::EDatabaseType::MySQL, - NYql::NConnector::NApi::EProtocol::NATIVE, + NYql::EGenericProtocol::NATIVE, "https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts", "200", R"({ @@ -590,7 +590,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { Test( NYql::EDatabaseType::MySQL, - NYql::NConnector::NApi::EProtocol::NATIVE, + NYql::EGenericProtocol::NATIVE, "https://mdb.api.cloud.yandex.net:443/managed-mysql/v1/clusters/etn021us5r9rhld1vgbh/hosts", "403", R"( @@ -624,7 +624,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { }; Test( NYql::EDatabaseType::DataStreams, - NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED, + NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED, "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod/database?databaseId=etn021us5r9rhld1vgbh", "403", R"( @@ -642,7 +642,7 @@ Y_UNIT_TEST_SUITE(TDatabaseResolverTests) { NYql::TDatabaseAuth databaseAuth; databaseAuth.UseTls = true; - databaseAuth.Protocol = NYql::NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED; + databaseAuth.Protocol = NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED; TString databaseId1{"etn021us5r9rhld1vgb1"}; TString databaseId2{"etn021us5r9rhld1vgb2"}; diff --git a/ydb/core/fq/libs/actors/ya.make b/ydb/core/fq/libs/actors/ya.make index 1b238dbfd9..cbb38d8335 100644 --- a/ydb/core/fq/libs/actors/ya.make +++ b/ydb/core/fq/libs/actors/ya.make @@ -72,7 +72,6 @@ PEERDIR( ydb/library/yql/providers/dq/provider ydb/library/yql/providers/dq/provider/exec ydb/library/yql/providers/dq/worker_manager/interface - ydb/library/yql/providers/generic/connector/api/common ydb/library/yql/providers/generic/connector/libcpp ydb/library/yql/providers/generic/provider ydb/library/yql/providers/pq/cm_client diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp index 8c0523189e..15d0c6522c 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp @@ -56,14 +56,14 @@ namespace NFq { ui32 port; switch (params.Protocol) { - case NYql::NConnector::NApi::EProtocol::NATIVE: + case NYql::EGenericProtocol::NATIVE: port = params.UseTls ? CLICKHOUSE_NATIVE_SECURE_PORT : CLICKHOUSE_NATIVE_INSECURE_PORT; break; - case NYql::NConnector::NApi::EProtocol::HTTP: + case NYql::EGenericProtocol::HTTP: port = params.UseTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT; break; default: - ythrow yexception() << "Unexpected protocol for ClickHouse: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); + ythrow yexception() << "Unexpected protocol for ClickHouse: " << NYql::EGenericProtocol_Name(params.Protocol); } return TEndpoint(fixedHost, port); @@ -71,26 +71,26 @@ namespace NFq { case NYql::EDatabaseType::PostgreSQL: // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect switch (params.Protocol) { - case NYql::NConnector::NApi::EProtocol::NATIVE: + case NYql::EGenericProtocol::NATIVE: return TEndpoint(fixedHost, POSTGRESQL_PORT); default: - ythrow yexception() << "Unexpected protocol for PostgreSQL " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); + ythrow yexception() << "Unexpected protocol for PostgreSQL " << NYql::EGenericProtocol_Name(params.Protocol); } case NYql::EDatabaseType::Greenplum: // https://cloud.yandex.ru/docs/managed-greenplum/operations/connect switch (params.Protocol) { - case NYql::NConnector::NApi::EProtocol::NATIVE: + case NYql::EGenericProtocol::NATIVE: return TEndpoint(fixedHost, GREENPLUM_PORT); default: - ythrow yexception() << "Unexpected protocol for Greenplum: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); + ythrow yexception() << "Unexpected protocol for Greenplum: " << NYql::EGenericProtocol_Name(params.Protocol); } case NYql::EDatabaseType::MySQL: // https://cloud.yandex.ru/docs/managed-mysql/operations/connect switch (params.Protocol) { - case NYql::NConnector::NApi::EProtocol::NATIVE: + case NYql::EGenericProtocol::NATIVE: return TEndpoint(fixedHost, MYSQL_PORT); default: - ythrow yexception() << "Unexpected protocol for MySQL: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); + ythrow yexception() << "Unexpected protocol for MySQL: " << NYql::EGenericProtocol_Name(params.Protocol); } default: ythrow yexception() << "Unexpected database type: " << ToString(params.DatabaseType); diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp index f02d3a4e08..f05a6e49aa 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp @@ -11,7 +11,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::ClickHouse, .MdbHost = "rc1c-p5waby2y5y1kb5ue.db.yandex.net", .UseTls = true, - .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + .Protocol = NYql::EGenericProtocol::HTTP, }; UNIT_ASSERT_VALUES_EQUAL( @@ -22,7 +22,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::ClickHouse, .MdbHost = "ya.ru", .UseTls = false, - .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + .Protocol = NYql::EGenericProtocol::HTTP, }; UNIT_ASSERT_VALUES_EQUAL( @@ -37,7 +37,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::ClickHouse, .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", .UseTls = true, - .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + .Protocol = NYql::EGenericProtocol::HTTP, }; UNIT_ASSERT_VALUES_EQUAL( @@ -48,7 +48,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::PostgreSQL, .MdbHost = "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", .UseTls = false, - .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + .Protocol = NYql::EGenericProtocol::NATIVE, }; UNIT_ASSERT_VALUES_EQUAL( @@ -65,7 +65,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::ClickHouse, .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", .UseTls = false, - .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + .Protocol = NYql::EGenericProtocol::HTTP, }; UNIT_ASSERT_VALUES_EQUAL( @@ -76,7 +76,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::ClickHouse, .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", .UseTls = false, - .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + .Protocol = NYql::EGenericProtocol::NATIVE, }; UNIT_ASSERT_VALUES_EQUAL( @@ -87,7 +87,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::ClickHouse, .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", .UseTls = true, - .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + .Protocol = NYql::EGenericProtocol::HTTP, }; UNIT_ASSERT_VALUES_EQUAL( @@ -98,7 +98,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::ClickHouse, .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", .UseTls = true, - .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + .Protocol = NYql::EGenericProtocol::NATIVE, }; UNIT_ASSERT_VALUES_EQUAL( @@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { .DatabaseType = NYql::EDatabaseType::PostgreSQL, .MdbHost = "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", .UseTls = true, - .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + .Protocol = NYql::EGenericProtocol::NATIVE, }; UNIT_ASSERT_VALUES_EQUAL( diff --git a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp index 0a5939a82c..480ad5f299 100644 --- a/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp @@ -35,7 +35,7 @@ namespace NKikimr::NKqp { Ydb, }; - NApi::TDataSourceInstance MakeDataSourceInstance(EProviderType providerType) { + NYql::TGenericDataSourceInstance MakeDataSourceInstance(EProviderType providerType) { switch (providerType) { case EProviderType::PostgreSQL: return TConnectorClientMock::TPostgreSQLDataSourceInstanceBuilder<>().GetResult(); @@ -96,7 +96,7 @@ namespace NKikimr::NKqp { // prepare mock auto clientMock = std::make_shared<TConnectorClientMock>(); - const NApi::TDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); + const NYql::TGenericDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); // step 1: DescribeTable // clang-format off @@ -192,7 +192,7 @@ namespace NKikimr::NKqp { // prepare mock auto clientMock = std::make_shared<TConnectorClientMock>(); - const NApi::TDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); + const NYql::TGenericDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); constexpr size_t ROWS_COUNT = 5; @@ -285,7 +285,7 @@ namespace NKikimr::NKqp { // prepare mock auto clientMock = std::make_shared<TConnectorClientMock>(); - const NApi::TDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); + const NYql::TGenericDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); constexpr size_t ROWS_COUNT = 5; @@ -374,7 +374,7 @@ namespace NKikimr::NKqp { // prepare mock auto clientMock = std::make_shared<TConnectorClientMock>(); - const NApi::TDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); + const NYql::TGenericDataSourceInstance dataSourceInstance = MakeDataSourceInstance(providerType); // clang-format off const NApi::TSelect select = TConnectorClientMock::TSelectBuilder<>() .DataSourceInstance(dataSourceInstance) diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h index 94237dd434..06ce03a48e 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h +++ b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h @@ -3,7 +3,7 @@ #include <library/cpp/threading/future/future.h> #include <util/string/builder.h> #include <util/string/cast.h> -#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> +#include <yql/essentials/providers/common/proto/gateways_config.pb.h> #include <yql/essentials/public/issue/yql_issue.h> namespace NYql { @@ -22,47 +22,47 @@ enum class EDatabaseType { Logging }; -inline EDatabaseType DatabaseTypeFromDataSourceKind(NConnector::NApi::EDataSourceKind dataSourceKind) { +inline EDatabaseType DatabaseTypeFromDataSourceKind(NYql::EGenericDataSourceKind dataSourceKind) { switch (dataSourceKind) { - case NConnector::NApi::EDataSourceKind::POSTGRESQL: + case NYql::EGenericDataSourceKind::POSTGRESQL: return EDatabaseType::PostgreSQL; - case NConnector::NApi::EDataSourceKind::CLICKHOUSE: + case NYql::EGenericDataSourceKind::CLICKHOUSE: return EDatabaseType::ClickHouse; - case NConnector::NApi::EDataSourceKind::YDB: + case NYql::EGenericDataSourceKind::YDB: return EDatabaseType::Ydb; - case NConnector::NApi::EDataSourceKind::MYSQL: + case NYql::EGenericDataSourceKind::MYSQL: return EDatabaseType::MySQL; - case NConnector::NApi::EDataSourceKind::GREENPLUM: + case NYql::EGenericDataSourceKind::GREENPLUM: return EDatabaseType::Greenplum; - case NConnector::NApi::EDataSourceKind::MS_SQL_SERVER: + case NYql::EGenericDataSourceKind::MS_SQL_SERVER: return EDatabaseType::MsSQLServer; - case NConnector::NApi::EDataSourceKind::ORACLE: + case NYql::EGenericDataSourceKind::ORACLE: return EDatabaseType::Oracle; - case NConnector::NApi::EDataSourceKind::LOGGING: + case NYql::EGenericDataSourceKind::LOGGING: return EDatabaseType::Logging; default: - ythrow yexception() << "Unknown data source kind: " << NConnector::NApi::EDataSourceKind_Name(dataSourceKind); + ythrow yexception() << "Unknown data source kind: " << NYql::EGenericDataSourceKind_Name(dataSourceKind); } } -inline NConnector::NApi::EDataSourceKind DatabaseTypeToDataSourceKind(EDatabaseType databaseType) { +inline NYql::EGenericDataSourceKind DatabaseTypeToDataSourceKind(EDatabaseType databaseType) { switch (databaseType) { case EDatabaseType::PostgreSQL: - return NConnector::NApi::EDataSourceKind::POSTGRESQL; + return NYql::EGenericDataSourceKind::POSTGRESQL; case EDatabaseType::ClickHouse: - return NConnector::NApi::EDataSourceKind::CLICKHOUSE; + return NYql::EGenericDataSourceKind::CLICKHOUSE; case EDatabaseType::Ydb: - return NConnector::NApi::EDataSourceKind::YDB; + return NYql::EGenericDataSourceKind::YDB; case EDatabaseType::MySQL: - return NConnector::NApi::EDataSourceKind::MYSQL; + return NYql::EGenericDataSourceKind::MYSQL; case EDatabaseType::Greenplum: - return NConnector::NApi::EDataSourceKind::GREENPLUM; + return NYql::EGenericDataSourceKind::GREENPLUM; case EDatabaseType::MsSQLServer: - return NConnector::NApi::EDataSourceKind::MS_SQL_SERVER; + return NYql::EGenericDataSourceKind::MS_SQL_SERVER; case EDatabaseType::Oracle: - return NConnector::NApi::EDataSourceKind::ORACLE; + return NYql::EGenericDataSourceKind::ORACLE; case EDatabaseType::Logging: - return NConnector::NApi::EDataSourceKind::LOGGING; + return NYql::EGenericDataSourceKind::LOGGING; default: ythrow yexception() << "Unknown database type: " << ToString(databaseType); } @@ -93,7 +93,7 @@ struct TDatabaseAuth { // For some of the data sources accessible via generic provider it's possible to specify the connection protocol. // This setting may impact the throughput. - NConnector::NApi::EProtocol Protocol = NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED; + NYql::EGenericProtocol Protocol = NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED; bool operator==(const TDatabaseAuth& other) const { return std::tie(StructuredToken, AddBearerToToken, UseTls, Protocol) == std::tie(other.StructuredToken, other.AddBearerToToken, other.UseTls, Protocol); diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h index abf38158aa..6eabe6df83 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h +++ b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h @@ -13,7 +13,7 @@ namespace NYql { NYql::EDatabaseType DatabaseType; TString MdbHost; bool UseTls; - NConnector::NApi::EProtocol Protocol; + NYql::EGenericProtocol Protocol; }; using TPtr = std::shared_ptr<IMdbEndpointGenerator>; diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make index e2a48a290c..48142fe1e9 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make +++ b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make @@ -7,7 +7,7 @@ SRCS( PEERDIR( library/cpp/threading/future - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto yql/essentials/public/issue ) diff --git a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp index c940fdbdbb..0208ccec1c 100644 --- a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp +++ b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp @@ -81,13 +81,13 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { runtime.Initialize(); auto edge = runtime.AllocateEdgeActor(); - NYql::NConnector::NApi::TDataSourceInstance dsi; - dsi.Setkind(NYql::NConnector::NApi::EDataSourceKind::YDB); + NYql::TGenericDataSourceInstance dsi; + dsi.Setkind(NYql::EGenericDataSourceKind::YDB); dsi.mutable_endpoint()->Sethost("some_host"); dsi.mutable_endpoint()->Setport(2135); dsi.Setdatabase("some_db"); dsi.Setuse_tls(true); - dsi.set_protocol(::NYql::NConnector::NApi::EProtocol::NATIVE); + dsi.set_protocol(::NYql::EGenericProtocol::NATIVE); auto token = dsi.mutable_credentials()->mutable_token(); token->Settype("IAM"); token->Setvalue("TEST_TOKEN"); diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index e1f3854b79..1792e18b0b 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -117,11 +117,11 @@ namespace NYql::NDq { void Bootstrap() { auto dsi = LookupSource.data_source_instance(); YQL_CLOG(INFO, ProviderGeneric) << "New generic proivider lookup source actor(ActorId=" << SelfId() << ") for" - << " kind=" << NYql::NConnector::NApi::EDataSourceKind_Name(dsi.kind()) + << " kind=" << NYql::EGenericDataSourceKind_Name(dsi.kind()) << ", endpoint=" << dsi.endpoint().ShortDebugString() << ", database=" << dsi.database() << ", use_tls=" << ToString(dsi.use_tls()) - << ", protocol=" << NYql::NConnector::NApi::EProtocol_Name(dsi.protocol()) + << ", protocol=" << NYql::EGenericProtocol_Name(dsi.protocol()) << ", table=" << LookupSource.table(); Become(&TGenericLookupActor::StateFunc); } diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index 5fc47f7178..28f59913cb 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -495,11 +495,11 @@ namespace NYql::NDq { { const auto dsi = source.select().data_source_instance(); YQL_CLOG(INFO, ProviderGeneric) << "Creating read actor with params:" - << " kind=" << NYql::NConnector::NApi::EDataSourceKind_Name(dsi.kind()) + << " kind=" << NYql::EGenericDataSourceKind_Name(dsi.kind()) << ", endpoint=" << dsi.endpoint().ShortDebugString() << ", database=" << dsi.database() << ", use_tls=" << ToString(dsi.use_tls()) - << ", protocol=" << NYql::NConnector::NApi::EProtocol_Name(dsi.protocol()); + << ", protocol=" << NYql::EGenericProtocol_Name(dsi.protocol()); // FIXME: strange piece of logic - authToken is created but not used: // https://a.yandex-team.ru/arcadia/ydb/library/yql/providers/clickhouse/actors/yql_ch_read_actor.cpp?rev=r11550199#L140 diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp index 5371cf93d6..753bb4c940 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.cpp @@ -27,7 +27,7 @@ namespace NYql::NDq { CredentialsProvider_ = credentialsProviderFactory->CreateProvider(); } - TString TGenericTokenProvider::MaybeFillToken(NConnector::NApi::TDataSourceInstance& dsi) const { + TString TGenericTokenProvider::MaybeFillToken(NYql::TGenericDataSourceInstance& dsi) const { // 1. Don't need tokens if basic auth is set if (dsi.credentials().has_basic()) { return {}; diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.h b/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.h index c60b610f1f..304398d4f4 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.h +++ b/ydb/library/yql/providers/generic/actors/yql_generic_token_provider.h @@ -22,7 +22,7 @@ namespace NYql::NDq { // MaybeFillToken sets IAM-token within DataSourceInstance. // Returns string containing error, if it happened. - TString MaybeFillToken(NConnector::NApi::TDataSourceInstance& dsi) const; + TString MaybeFillToken(NYql::TGenericDataSourceInstance& dsi) const; private: TString StaticIAMToken_; diff --git a/ydb/library/yql/providers/generic/connector/api/service/connector.proto b/ydb/library/yql/providers/generic/connector/api/service/connector.proto index ec1b855ff3..e70c8ab227 100644 --- a/ydb/library/yql/providers/generic/connector/api/service/connector.proto +++ b/ydb/library/yql/providers/generic/connector/api/service/connector.proto @@ -4,7 +4,7 @@ package NYql.NConnector.NApi; import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto"; -option go_package = "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service"; +option go_package = "a.yandex-team.ru/contrib/ydb/library/yql/providers/generic/connector/libgo/service"; // Connector provides unified interface for various data sources that can be used to extend capabilities // of YQ and YQL services. diff --git a/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto index 1d638fab12..a91d3eb217 100644 --- a/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto @@ -5,16 +5,16 @@ package NYql.NConnector.NApi; import "ydb/public/api/protos/ydb_value.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; -import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto"; +import "yql/essentials/providers/common/proto/gateways_config.proto"; -option go_package = "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos"; +option go_package = "a.yandex-team.ru/contrib/ydb/library/yql/providers/generic/connector/libgo/service/protos"; // ---------- API Requests ---------- // TListTablesRequest requests the list of tables in a particular database of the data source message TListTablesRequest { // Data source instance to connect - TDataSourceInstance data_source_instance = 1; + NYql.TGenericDataSourceInstance data_source_instance = 1; // There may be a huge number of tables in the data source, // and here are ways to extract only necessary ones: @@ -36,7 +36,7 @@ message TListTablesResponse { // TDescribeTableRequest requests table metadata message TDescribeTableRequest { // Data source instance to connect - TDataSourceInstance data_source_instance = 1; + NYql.TGenericDataSourceInstance data_source_instance = 1; // Table to describe string table = 2; // Rules for type mapping @@ -115,7 +115,7 @@ message TListSplitsResponse { // TODO: support JOIN, ORDER BY, GROUP BY message TSelect { // Data source instance to connect - TDataSourceInstance data_source_instance = 1; + NYql.TGenericDataSourceInstance data_source_instance = 1; // Describes what particularly to get from the data source message TWhat { @@ -200,7 +200,7 @@ message TReadSplitsRequest { // Data source instance to connect. // Deprecated field: server implementations must rely on // TDataSourceInstance provided in each TSelect. - TDataSourceInstance data_source_instance = 1 [deprecated = true]; + NYql.TGenericDataSourceInstance data_source_instance = 1 [deprecated = true]; // Splits that YQ engine would like to read. repeated TSplit splits = 2; diff --git a/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make b/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make index f9fe5ea3ca..c05760b5e6 100644 --- a/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make @@ -1,7 +1,7 @@ PROTO_LIBRARY() PEERDIR( - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/public/api/protos ) diff --git a/ydb/library/yql/providers/generic/connector/api/ya.make b/ydb/library/yql/providers/generic/connector/api/ya.make index 254cdc978c..b21d3fa468 100644 --- a/ydb/library/yql/providers/generic/connector/api/ya.make +++ b/ydb/library/yql/providers/generic/connector/api/ya.make @@ -1,4 +1,3 @@ RECURSE( - common # Keep this library pure of dependencies on YDB protofiles service # Use YDB protofiles only here ) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp index 8227b03b35..ed92d42b65 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp @@ -32,7 +32,7 @@ namespace NYql::NConnector::NTest { void CreatePostgreSQLExternalDataSource( const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr, const TString& dataSourceName, - NApi::EProtocol protocol, + NYql::EGenericProtocol protocol, const TString& host, int port, const TString& login, @@ -65,7 +65,7 @@ namespace NYql::NConnector::NTest { "login"_a = login, "password"_a = password, "use_tls"_a = useTls ? "TRUE" : "FALSE", - "protocol"_a = NApi::EProtocol_Name(protocol), + "protocol"_a = NYql::EGenericProtocol_Name(protocol), "source_type"_a = PG_SOURCE_TYPE, "database"_a = databaseName, "schema"_a = schema); @@ -76,7 +76,7 @@ namespace NYql::NConnector::NTest { void CreateClickHouseExternalDataSource( const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr, const TString& dataSourceName, - NApi::EProtocol protocol, + NYql::EGenericProtocol protocol, const TString& clickHouseClusterId, const TString& login, const TString& password, @@ -110,7 +110,7 @@ namespace NYql::NConnector::NTest { "login"_a = login, "password"_a = password, "use_tls"_a = useTls ? "TRUE" : "FALSE", - "protocol"_a = NYql::NConnector::NApi::EProtocol_Name(protocol), + "protocol"_a = NYql::EGenericProtocol_Name(protocol), "service_account_id"_a = serviceAccountId, "service_account_id_signature"_a = serviceAccountIdSignature, "source_type"_a = ToString(NYql::EDatabaseType::ClickHouse), diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h index 7f76461159..0cec2f06ab 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h @@ -2,7 +2,7 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/core/formats/arrow/serializer/abstract.h> -#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> +#include <yql/essentials/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/generic/connector/libcpp/client.h> #include <ydb/library/yql/providers/generic/connector/libcpp/error.h> #include <ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h> @@ -41,7 +41,7 @@ namespace NYql::NConnector::NTest { } #define DATA_SOURCE_INSTANCE_SUBBUILDER() \ - TBuilder& DataSourceInstance(const NApi::TDataSourceInstance& proto) { \ + TBuilder& DataSourceInstance(const NYql::TGenericDataSourceInstance& proto) { \ this->Result_->mutable_data_source_instance()->CopyFrom(proto); \ return static_cast<TBuilder&>(*this); \ } \ @@ -184,7 +184,7 @@ namespace NYql::NConnector::NTest { void CreatePostgreSQLExternalDataSource( const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr, const TString& dataSourceName = DEFAULT_DATA_SOURCE_NAME, - NApi::EProtocol protocol = DEFAULT_PG_PROTOCOL, + NYql::EGenericProtocol protocol = DEFAULT_PG_PROTOCOL, const TString& host = DEFAULT_PG_HOST, int port = DEFAULT_PG_PORT, const TString& login = DEFAULT_LOGIN, @@ -196,7 +196,7 @@ namespace NYql::NConnector::NTest { void CreateClickHouseExternalDataSource( const std::shared_ptr<NKikimr::NKqp::TKikimrRunner>& kikimr, const TString& dataSourceName = DEFAULT_DATA_SOURCE_NAME, - NApi::EProtocol protocol = DEFAULT_CH_PROTOCOL, + NYql::EGenericProtocol protocol = DEFAULT_CH_PROTOCOL, const TString& clickHouseClusterId = DEFAULT_CH_CLUSTER_ID, const TString& login = DEFAULT_LOGIN, const TString& password = DEFAULT_PASSWORD, @@ -225,11 +225,11 @@ namespace NYql::NConnector::NTest { // template <class TDerived, class TParent = void /* no parent by default */> - struct TBaseDataSourceInstanceBuilder: public TProtoBuilder<TParent, NApi::TDataSourceInstance> { + struct TBaseDataSourceInstanceBuilder: public TProtoBuilder<TParent, NYql::TGenericDataSourceInstance> { using TBuilder = TDerived; - explicit TBaseDataSourceInstanceBuilder(NApi::TDataSourceInstance* result = nullptr, TParent* parent = nullptr) - : TProtoBuilder<TParent, NApi::TDataSourceInstance>(result, parent) + explicit TBaseDataSourceInstanceBuilder(NYql::TGenericDataSourceInstance* result = nullptr, TParent* parent = nullptr) + : TProtoBuilder<TParent, NYql::TGenericDataSourceInstance>(result, parent) { } @@ -256,7 +256,7 @@ namespace NYql::NConnector::NTest { struct TPostgreSQLDataSourceInstanceBuilder: public TBaseDataSourceInstanceBuilder<TPostgreSQLDataSourceInstanceBuilder<TParent>, TParent> { using TBase = TBaseDataSourceInstanceBuilder<TPostgreSQLDataSourceInstanceBuilder<TParent>, TParent>; - explicit TPostgreSQLDataSourceInstanceBuilder(NApi::TDataSourceInstance* result = nullptr, TParent* parent = nullptr) + explicit TPostgreSQLDataSourceInstanceBuilder(NYql::TGenericDataSourceInstance* result = nullptr, TParent* parent = nullptr) : TBase(result, parent) { FillWithDefaults(); @@ -266,7 +266,7 @@ namespace NYql::NConnector::NTest { TBase::FillWithDefaults(); this->Host(DEFAULT_PG_HOST); this->Port(DEFAULT_PG_PORT); - this->Kind(NApi::EDataSourceKind::POSTGRESQL); + this->Kind(NYql::EGenericDataSourceKind::POSTGRESQL); this->Protocol(DEFAULT_PG_PROTOCOL); this->Schema(DEFAULT_PG_SCHEMA); } @@ -276,7 +276,7 @@ namespace NYql::NConnector::NTest { struct TClickHouseDataSourceInstanceBuilder: public TBaseDataSourceInstanceBuilder<TClickHouseDataSourceInstanceBuilder<TParent>, TParent> { using TBase = TBaseDataSourceInstanceBuilder<TClickHouseDataSourceInstanceBuilder<TParent>, TParent>; - explicit TClickHouseDataSourceInstanceBuilder(NApi::TDataSourceInstance* result = nullptr, TParent* parent = nullptr) + explicit TClickHouseDataSourceInstanceBuilder(NYql::TGenericDataSourceInstance* result = nullptr, TParent* parent = nullptr) : TBase(result, parent) { FillWithDefaults(); @@ -286,7 +286,7 @@ namespace NYql::NConnector::NTest { TBase::FillWithDefaults(); this->Host(DEFAULT_CH_HOST); this->Port(DEFAULT_CH_PORT); - this->Kind(NApi::EDataSourceKind::CLICKHOUSE); + this->Kind(NYql::EGenericDataSourceKind::CLICKHOUSE); this->Protocol(DEFAULT_CH_PROTOCOL); } }; @@ -295,7 +295,7 @@ namespace NYql::NConnector::NTest { struct TYdbDataSourceInstanceBuilder: public TBaseDataSourceInstanceBuilder<TYdbDataSourceInstanceBuilder<TParent>, TParent> { using TBase = TBaseDataSourceInstanceBuilder<TYdbDataSourceInstanceBuilder<TParent>, TParent>; - explicit TYdbDataSourceInstanceBuilder(NApi::TDataSourceInstance* result = nullptr, TParent* parent = nullptr) + explicit TYdbDataSourceInstanceBuilder(NYql::TGenericDataSourceInstance* result = nullptr, TParent* parent = nullptr) : TBase(result, parent) { FillWithDefaults(); @@ -305,7 +305,7 @@ namespace NYql::NConnector::NTest { TBase::FillWithDefaults(); this->Host(DEFAULT_YDB_HOST); this->Port(DEFAULT_YDB_PORT); - this->Kind(NApi::EDataSourceKind::YDB); + this->Kind(NYql::EGenericDataSourceKind::YDB); this->Protocol(DEFAULT_YDB_PROTOCOL); } }; diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h index bbca9127a4..3cc9bd3950 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h @@ -1,5 +1,5 @@ #pragma once -#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> +#include <yql/essentials/providers/common/proto/gateways_config.pb.h> #include <util/generic/string.h> #include <util/string/builder.h> @@ -15,19 +15,19 @@ namespace NYql::NConnector::NTest { constexpr int DEFAULT_PG_PORT = 5432; constexpr bool DEFAULT_USE_TLS = true; extern const TString PG_SOURCE_TYPE; - constexpr NApi::EProtocol DEFAULT_PG_PROTOCOL = NApi::EProtocol::NATIVE; + constexpr NYql::EGenericProtocol DEFAULT_PG_PROTOCOL = NYql::EGenericProtocol::NATIVE; extern const TString DEFAULT_PG_SCHEMA; extern const TString DEFAULT_CH_HOST; constexpr int DEFAULT_CH_PORT = 8443; extern const TString DEFAULT_CH_ENDPOINT; extern const TString DEFAULT_CH_CLUSTER_ID; - constexpr NApi::EProtocol DEFAULT_CH_PROTOCOL = NApi::EProtocol::HTTP; + constexpr NYql::EGenericProtocol DEFAULT_CH_PROTOCOL = NYql::EGenericProtocol::HTTP; extern const TString DEFAULT_CH_SERVICE_ACCOUNT_ID; extern const TString DEFAULT_CH_SERVICE_ACCOUNT_ID_SIGNATURE; extern const TString DEFAULT_YDB_HOST; constexpr int DEFAULT_YDB_PORT = 2136; extern const TString DEFAULT_YDB_ENDPOINT; - constexpr NApi::EProtocol DEFAULT_YDB_PROTOCOL = NApi::EProtocol::NATIVE; + constexpr NYql::EGenericProtocol DEFAULT_YDB_PROTOCOL = NYql::EGenericProtocol::NATIVE; } // namespace NYql::NConnector::NTest diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/ya.make b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/ya.make index 9bfc3b305e..ef40fa52d6 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/ya.make +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/ya.make @@ -14,7 +14,7 @@ PEERDIR( ydb/core/kqp/ut/common ydb/library/yql/providers/common/db_id_async_resolver yql/essentials/providers/common/structured_token - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/libcpp ) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ya.make b/ydb/library/yql/providers/generic/connector/libcpp/ya.make index 5b88a539d9..d31e47403a 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ya.make +++ b/ydb/library/yql/providers/generic/connector/libcpp/ya.make @@ -14,7 +14,7 @@ PEERDIR( yql/essentials/ast ydb/library/yql/dq/actors/protos yql/essentials/providers/common/proto - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/api/service ydb/library/yql/providers/generic/connector/api/service/protos yql/essentials/public/issue diff --git a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make index 143966c5cd..db6c24486d 100644 --- a/ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make @@ -13,7 +13,7 @@ PY_SRCS( ) PEERDIR( - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/utils ydb/library/yql/providers/generic/connector/api/service/protos ydb/public/api/protos diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make index dde9f62942..3a8f114c35 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make @@ -65,7 +65,7 @@ TEST_SRCS( PEERDIR( contrib/python/pytest - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/api/service/protos ydb/library/yql/providers/generic/connector/tests/common_test_cases ydb/library/yql/providers/generic/connector/tests/utils diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make index b826eab127..18f425f9f3 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make @@ -65,7 +65,7 @@ TEST_SRCS( PEERDIR( contrib/python/pytest - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/common_test_cases ydb/library/yql/providers/generic/connector/tests/utils ydb/library/yql/providers/generic/connector/tests/utils/run diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make index 9fc3764f7d..70acbf909e 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make @@ -65,7 +65,7 @@ TEST_SRCS( PEERDIR( contrib/python/pytest - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/common_test_cases ydb/library/yql/providers/generic/connector/tests/utils ydb/library/yql/providers/generic/connector/tests/utils/run diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make index e34e0d5322..11a157241f 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make @@ -65,7 +65,7 @@ TEST_SRCS( PEERDIR( contrib/python/pytest - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/common_test_cases ydb/library/yql/providers/generic/connector/tests/utils ydb/library/yql/providers/generic/connector/tests/utils/run diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make index 0a9ed43f26..679a0dcd05 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make @@ -66,7 +66,7 @@ TEST_SRCS( PEERDIR( contrib/python/pytest - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/common_test_cases ydb/library/yql/providers/generic/connector/tests/utils ydb/library/yql/providers/generic/connector/tests/utils/run diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make index e2e749f8a5..d9fbc295a2 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make @@ -64,7 +64,7 @@ TEST_SRCS( PEERDIR( contrib/python/pytest - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/common_test_cases ydb/library/yql/providers/generic/connector/tests/utils ydb/library/yql/providers/generic/connector/tests/utils/run diff --git a/ydb/library/yql/providers/generic/connector/tests/join/ya.make b/ydb/library/yql/providers/generic/connector/tests/join/ya.make index 1bb206bc3c..cb8b8a6251 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/join/ya.make @@ -65,7 +65,7 @@ TEST_SRCS( PEERDIR( contrib/python/pytest - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/api/service/protos ydb/library/yql/providers/generic/connector/tests/common_test_cases ydb/library/yql/providers/generic/connector/tests/utils diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make index 362ed82c97..8cca116ec0 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make @@ -17,7 +17,7 @@ PY_SRCS( PEERDIR( contrib/python/Jinja2 contrib/python/PyYAML - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/api/service/protos ydb/library/yql/providers/generic/connector/tests/utils ydb/public/api/protos diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/ya.make index d0dea697ee..6cb533c609 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/utils/ya.make @@ -21,7 +21,7 @@ ENDIF() PEERDIR( contrib/python/PyYAML - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/utils/types ydb/public/api/protos yt/python/yt/yson diff --git a/ydb/library/yql/providers/generic/proto/source.proto b/ydb/library/yql/providers/generic/proto/source.proto index a78d4ddc27..a9a92d92e3 100644 --- a/ydb/library/yql/providers/generic/proto/source.proto +++ b/ydb/library/yql/providers/generic/proto/source.proto @@ -5,7 +5,7 @@ option cc_enable_arenas = true; package NYql.Generic; import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto"; -import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto"; +import "yql/essentials/providers/common/proto/gateways_config.proto"; message TSource { // Prepared Select expression @@ -25,7 +25,7 @@ message TSource { //TODO(YQ-3093) move SA creds to some common message message TLookupSource { - NYql.NConnector.NApi.TDataSourceInstance data_source_instance = 1; + NYql.TGenericDataSourceInstance data_source_instance = 1; string table = 2; // Credentials used to access managed databases APIs. // When working with external data source instances deployed in clouds, diff --git a/ydb/library/yql/providers/generic/proto/ya.make b/ydb/library/yql/providers/generic/proto/ya.make index 39a31b0406..31880aadde 100644 --- a/ydb/library/yql/providers/generic/proto/ya.make +++ b/ydb/library/yql/providers/generic/proto/ya.make @@ -4,7 +4,7 @@ ONLY_TAGS(CPP_PROTO) PEERDIR( ydb/library/yql/providers/generic/connector/api/service/protos - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ) SRCS( diff --git a/ydb/library/yql/providers/generic/provider/ut/pushdown/pushdown_ut.cpp b/ydb/library/yql/providers/generic/provider/ut/pushdown/pushdown_ut.cpp index 73e538e024..16cb4ac140 100644 --- a/ydb/library/yql/providers/generic/provider/ut/pushdown/pushdown_ut.cpp +++ b/ydb/library/yql/providers/generic/provider/ut/pushdown/pushdown_ut.cpp @@ -230,13 +230,13 @@ struct TPushdownFixture: public NUnitTest::TBaseFixture { auto* cluster = GatewaysCfg.MutableGeneric()->AddClusterMapping(); cluster->SetName("test_cluster"); - cluster->SetKind(NConnector::NApi::POSTGRESQL); + cluster->SetKind(NYql::EGenericDataSourceKind::POSTGRESQL); cluster->MutableEndpoint()->set_host("host"); cluster->MutableEndpoint()->set_port(42); cluster->MutableCredentials()->mutable_basic()->set_username("user"); cluster->MutableCredentials()->mutable_basic()->set_password("password"); cluster->SetDatabaseName("database"); - cluster->SetProtocol(NConnector::NApi::NATIVE); + cluster->SetProtocol(NYql::EGenericProtocol::NATIVE); } GenericState = MakeIntrusive<TGenericState>( diff --git a/ydb/library/yql/providers/generic/provider/ya.make b/ydb/library/yql/providers/generic/provider/ya.make index b87b656e56..5ee4ed066a 100644 --- a/ydb/library/yql/providers/generic/provider/ya.make +++ b/ydb/library/yql/providers/generic/provider/ya.make @@ -54,7 +54,7 @@ PEERDIR( ydb/library/yql/providers/dq/expr_nodes ydb/library/yql/providers/generic/expr_nodes ydb/library/yql/providers/generic/proto - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/libcpp yql/essentials/providers/result/expr_nodes ydb/library/yql/utils/plan diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp index 9745f2c594..ce35300b49 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp @@ -6,7 +6,7 @@ #include <util/string/cast.h> #include <util/generic/yexception.h> -#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> +#include <yql/essentials/providers/common/proto/gateways_config.pb.h> #include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> #include "yql_generic_cluster_config.h" @@ -200,25 +200,23 @@ namespace NYql { void ParseProtocol(const THashMap<TString, TString>& properties, NYql::TGenericClusterConfig& clusterConfig) { - using namespace NConnector::NApi; - // For some datasources the PROTOCOL is not required - if (clusterConfig.GetKind() == EDataSourceKind::LOGGING) { - clusterConfig.SetProtocol(EProtocol::PROTOCOL_UNSPECIFIED); + if (clusterConfig.GetKind() == EGenericDataSourceKind::LOGGING) { + clusterConfig.SetProtocol(EGenericProtocol::PROTOCOL_UNSPECIFIED); return; } // For the most of transactional databases the PROTOCOL is always NATIVE if (IsIn({ - EDataSourceKind::GREENPLUM, - EDataSourceKind::YDB, - EDataSourceKind::MYSQL, - EDataSourceKind::MS_SQL_SERVER, - EDataSourceKind::ORACLE, + EGenericDataSourceKind::GREENPLUM, + EGenericDataSourceKind::YDB, + EGenericDataSourceKind::MYSQL, + EGenericDataSourceKind::MS_SQL_SERVER, + EGenericDataSourceKind::ORACLE, }, clusterConfig.GetKind() )) { - clusterConfig.SetProtocol(EProtocol::NATIVE); + clusterConfig.SetProtocol(EGenericProtocol::NATIVE); return; } @@ -231,13 +229,13 @@ namespace NYql { auto input = it->second; input.to_upper(); - EProtocol protocol; - if (!EProtocol_Parse(input, &protocol)) { + EGenericProtocol protocol; + if (!NYql::EGenericProtocol_Parse(input, &protocol)) { TStringBuilder b; b << "invalid 'PROTOCOL' value: '" << it->second << "', valid types are: "; - for (auto i = EProtocol_MIN + 1; i < EProtocol_MAX; i++) { - b << EProtocol_Name(i); - if (i != EProtocol_MAX - 1) { + for (auto i = EGenericProtocol_MIN + 1; i < EGenericProtocol_MAX; i++) { + b << NYql::EGenericProtocol_Name(i); + if (i != EGenericProtocol_MAX - 1) { b << ", "; } } @@ -351,7 +349,7 @@ namespace NYql { "context"_a = context, "msg"_a = msg, "name"_a = clusterConfig.GetName(), - "kind"_a = NConnector::NApi::EDataSourceKind_Name(clusterConfig.GetKind()), + "kind"_a = NYql::EGenericDataSourceKind_Name(clusterConfig.GetKind()), "host"_a = clusterConfig.GetEndpoint().Gethost(), "port"_a = clusterConfig.GetEndpoint().Getport(), "database_id"_a = clusterConfig.GetDatabaseId(), @@ -362,31 +360,31 @@ namespace NYql { "token"_a = ToString(clusterConfig.GetToken().size()), "use_ssl"_a = clusterConfig.GetUseSsl() ? "TRUE" : "FALSE", "database_name"_a = clusterConfig.GetDatabaseName(), - "protocol"_a = NConnector::NApi::EProtocol_Name(clusterConfig.GetProtocol()), + "protocol"_a = NYql::EGenericProtocol_Name(clusterConfig.GetProtocol()), "schema"_a = GetPropertyWithDefault(clusterConfig.datasourceoptions(), "schema"), "folder_id"_a = GetPropertyWithDefault(clusterConfig.datasourceoptions(), "folder_id") ); } - static const TSet<NConnector::NApi::EDataSourceKind> managedDatabaseKinds{ - NConnector::NApi::EDataSourceKind::CLICKHOUSE, - NConnector::NApi::EDataSourceKind::GREENPLUM, - NConnector::NApi::EDataSourceKind::MYSQL, - NConnector::NApi::EDataSourceKind::POSTGRESQL, - NConnector::NApi::EDataSourceKind::YDB, + static const TSet<NYql::EGenericDataSourceKind> managedDatabaseKinds{ + NYql::EGenericDataSourceKind::CLICKHOUSE, + NYql::EGenericDataSourceKind::GREENPLUM, + NYql::EGenericDataSourceKind::MYSQL, + NYql::EGenericDataSourceKind::POSTGRESQL, + NYql::EGenericDataSourceKind::YDB, }; - static const TSet<NConnector::NApi::EDataSourceKind> traditionalRelationalDatabaseKinds{ - NConnector::NApi::EDataSourceKind::CLICKHOUSE, - NConnector::NApi::EDataSourceKind::GREENPLUM, - NConnector::NApi::EDataSourceKind::MS_SQL_SERVER, - NConnector::NApi::EDataSourceKind::MYSQL, - NConnector::NApi::EDataSourceKind::ORACLE, - NConnector::NApi::EDataSourceKind::POSTGRESQL, + static const TSet<NYql::EGenericDataSourceKind> traditionalRelationalDatabaseKinds{ + NYql::EGenericDataSourceKind::CLICKHOUSE, + NYql::EGenericDataSourceKind::GREENPLUM, + NYql::EGenericDataSourceKind::MS_SQL_SERVER, + NYql::EGenericDataSourceKind::MYSQL, + NYql::EGenericDataSourceKind::ORACLE, + NYql::EGenericDataSourceKind::POSTGRESQL, }; - bool DataSourceMustHaveDataBaseName(const NConnector::NApi::EDataSourceKind& sourceKind) { - return traditionalRelationalDatabaseKinds.contains(sourceKind) && sourceKind != NConnector::NApi::ORACLE; + bool DataSourceMustHaveDataBaseName(const NYql::EGenericDataSourceKind& sourceKind) { + return traditionalRelationalDatabaseKinds.contains(sourceKind) && sourceKind != NYql::EGenericDataSourceKind::ORACLE; } void ValidateGenericClusterConfig( @@ -447,7 +445,7 @@ namespace NYql { // YDB: // * set database name when working with on-prem YDB instance; // * but set database ID when working with managed YDB. - if (clusterConfig.GetKind() == NConnector::NApi::YDB) { + if (clusterConfig.GetKind() == NYql::EGenericDataSourceKind::YDB) { if (clusterConfig.HasDatabaseName() && clusterConfig.HasDatabaseId()) { return ValidationError( clusterConfig, @@ -465,7 +463,7 @@ namespace NYql { // Oracle: // * always set service_name for oracle; - if (clusterConfig.GetKind() == NConnector::NApi::ORACLE) { + if (clusterConfig.GetKind() == NYql::EGenericDataSourceKind::ORACLE) { if (!clusterConfig.GetDataSourceOptions().contains("service_name")) { return ValidationError( clusterConfig, @@ -490,7 +488,7 @@ namespace NYql { return ValidationError(clusterConfig, context, "empty field 'Name'"); } - if (clusterConfig.GetKind() == NConnector::NApi::EDataSourceKind::DATA_SOURCE_KIND_UNSPECIFIED) { + if (clusterConfig.GetKind() == NYql::EGenericDataSourceKind::DATA_SOURCE_KIND_UNSPECIFIED) { return ValidationError(clusterConfig, context, "empty field 'Kind'"); } diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp index 6ec38109bf..c36536b6fd 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp @@ -21,23 +21,23 @@ namespace NYql { namespace { - TString GetSourceType(NYql::NConnector::NApi::TDataSourceInstance dsi) { + TString GetSourceType(NYql::TGenericDataSourceInstance dsi) { switch (dsi.kind()) { - case NYql::NConnector::NApi::CLICKHOUSE: + case NYql::EGenericDataSourceKind::CLICKHOUSE: return "ClickHouseGeneric"; - case NYql::NConnector::NApi::POSTGRESQL: + case NYql::EGenericDataSourceKind::POSTGRESQL: return "PostgreSqlGeneric"; - case NYql::NConnector::NApi::MYSQL: + case NYql::EGenericDataSourceKind::MYSQL: return "MySqlGeneric"; - case NYql::NConnector::NApi::YDB: + case NYql::EGenericDataSourceKind::YDB: return "YdbGeneric"; - case NYql::NConnector::NApi::GREENPLUM: + case NYql::EGenericDataSourceKind::GREENPLUM: return "GreenplumGeneric"; - case NYql::NConnector::NApi::MS_SQL_SERVER: + case NYql::EGenericDataSourceKind::MS_SQL_SERVER: return "MsSQLServerGeneric"; - case NYql::NConnector::NApi::ORACLE: + case NYql::EGenericDataSourceKind::ORACLE: return "OracleGeneric"; - case NYql::NConnector::NApi::LOGGING: + case NYql::EGenericDataSourceKind::LOGGING: return "LoggingGeneric"; default: ythrow yexception() << "Data source kind is unknown or not specified"; @@ -167,7 +167,7 @@ namespace NYql { // Managed YDB (including YDB underlying Logging) supports access via IAM token. // If exist, copy service account creds to obtain tokens during request execution phase. // If exists, copy previously created token. - if (IsIn({NConnector::NApi::EDataSourceKind::YDB, NConnector::NApi::EDataSourceKind::LOGGING}, clusterConfig.kind())) { + if (IsIn({NYql::EGenericDataSourceKind::YDB, NYql::EGenericDataSourceKind::LOGGING}, clusterConfig.kind())) { source.SetServiceAccountId(clusterConfig.GetServiceAccountId()); source.SetServiceAccountIdSignature(clusterConfig.GetServiceAccountIdSignature()); source.SetToken(State_->Types->Credentials->FindCredentialContent( @@ -198,36 +198,36 @@ namespace NYql { properties["Table"] = table; auto [tableMeta, issue] = State_->GetTable(clusterName, table); if (!issue) { - const NConnector::NApi::TDataSourceInstance& dataSourceInstance = tableMeta.value()->DataSourceInstance; + const NYql::TGenericDataSourceInstance& dataSourceInstance = tableMeta.value()->DataSourceInstance; switch (dataSourceInstance.kind()) { - case NConnector::NApi::CLICKHOUSE: + case NYql::EGenericDataSourceKind::CLICKHOUSE: properties["SourceType"] = "ClickHouse"; break; - case NConnector::NApi::POSTGRESQL: + case NYql::EGenericDataSourceKind::POSTGRESQL: properties["SourceType"] = "PostgreSql"; break; - case NConnector::NApi::MYSQL: + case NYql::EGenericDataSourceKind::MYSQL: properties["SourceType"] = "MySql"; break; - case NConnector::NApi::YDB: + case NYql::EGenericDataSourceKind::YDB: properties["SourceType"] = "Ydb"; break; - case NConnector::NApi::GREENPLUM: + case NYql::EGenericDataSourceKind::GREENPLUM: properties["SourceType"] = "Greenplum"; break; - case NConnector::NApi::MS_SQL_SERVER: + case NYql::EGenericDataSourceKind::MS_SQL_SERVER: properties["SourceType"] = "MsSQLServer"; break; - case NConnector::NApi::ORACLE: + case NYql::EGenericDataSourceKind::ORACLE: properties["SourceType"] = "Oracle"; break; - case NConnector::NApi::LOGGING: + case NYql::EGenericDataSourceKind::LOGGING: properties["SourceType"] = "Logging"; break; - case NConnector::NApi::DATA_SOURCE_KIND_UNSPECIFIED: + case NYql::EGenericDataSourceKind::DATA_SOURCE_KIND_UNSPECIFIED: break; default: - properties["SourceType"] = NConnector::NApi::EDataSourceKind_Name(dataSourceInstance.kind()); + properties["SourceType"] = NYql::EGenericDataSourceKind_Name(dataSourceInstance.kind()); break; } @@ -236,15 +236,15 @@ namespace NYql { } switch (dataSourceInstance.protocol()) { - case NConnector::NApi::NATIVE: + case NYql::EGenericProtocol::NATIVE: properties["Protocol"] = "Native"; break; - case NConnector::NApi::HTTP: + case NYql::EGenericProtocol::HTTP: properties["Protocol"] = "Http"; break; - case NConnector::NApi::PROTOCOL_UNSPECIFIED: + case NYql::EGenericProtocol::PROTOCOL_UNSPECIFIED: default: - properties["Protocol"] = NConnector::NApi::EProtocol_Name(dataSourceInstance.protocol()); + properties["Protocol"] = NYql::EGenericProtocol_Name(dataSourceInstance.protocol()); break; } } @@ -291,7 +291,7 @@ namespace NYql { // Managed YDB supports access via IAM token. // If exist, copy service account creds to obtain tokens during request execution phase. // If exists, copy previously created token. - if (clusterConfig.kind() == NConnector::NApi::EDataSourceKind::YDB) { + if (clusterConfig.kind() == NYql::EGenericDataSourceKind::YDB) { source.SetServiceAccountId(clusterConfig.GetServiceAccountId()); source.SetServiceAccountIdSignature(clusterConfig.GetServiceAccountIdSignature()); source.SetToken(State_->Types->Credentials->FindCredentialContent( diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp index 95f5b3f7fa..0a3f8bd00d 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp @@ -174,7 +174,7 @@ namespace NYql { // If we work with managed YDB, we find out database name // only after database id (== cluster id) resolving. - if (clusterConfigIter->second.kind() == NConnector::NApi::EDataSourceKind::YDB) { + if (clusterConfigIter->second.kind() == NYql::EGenericDataSourceKind::YDB) { clusterConfigIter->second.set_databasename(databaseDescription.Database); } diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp index 9d8b669c83..71d3e97736 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp @@ -28,7 +28,7 @@ namespace NYql { struct TGenericTableDescription { using TPtr = std::shared_ptr<TGenericTableDescription>; - NConnector::NApi::TDataSourceInstance DataSourceInstance; + NYql::TGenericDataSourceInstance DataSourceInstance; std::optional<NConnector::NApi::TDescribeTableResponse> Response; }; @@ -328,14 +328,14 @@ namespace NYql { request.set_schema(schema); } - void SetOracleServiceName(NYql::NConnector::NApi::TOracleDataSourceOptions& options, const TGenericClusterConfig& clusterConfig) { + void SetOracleServiceName(NYql::TOracleDataSourceOptions& options, const TGenericClusterConfig& clusterConfig) { const auto it = clusterConfig.GetDataSourceOptions().find("service_name"); if (it != clusterConfig.GetDataSourceOptions().end()) { options.set_service_name(it->second); } } - void SetLoggingFolderId(NYql::NConnector::NApi::TLoggingDataSourceOptions& options, const TGenericClusterConfig& clusterConfig) { + void SetLoggingFolderId(NYql::TLoggingDataSourceOptions& options, const TGenericClusterConfig& clusterConfig) { const auto it = clusterConfig.GetDataSourceOptions().find("folder_id"); if (it != clusterConfig.GetDataSourceOptions().end()) { options.set_folder_id(it->second); @@ -345,32 +345,32 @@ namespace NYql { void FillDataSourceOptions(NConnector::NApi::TDescribeTableRequest& request, const TGenericClusterConfig& clusterConfig) { const auto dataSourceKind = clusterConfig.GetKind(); switch (dataSourceKind) { - case NYql::NConnector::NApi::CLICKHOUSE: + case NYql::EGenericDataSourceKind::CLICKHOUSE: break; - case NYql::NConnector::NApi::YDB: + case NYql::EGenericDataSourceKind::YDB: break; - case NYql::NConnector::NApi::MYSQL: + case NYql::EGenericDataSourceKind::MYSQL: break; - case NYql::NConnector::NApi::GREENPLUM: { + case NYql::EGenericDataSourceKind::GREENPLUM: { auto* options = request.mutable_data_source_instance()->mutable_gp_options(); SetSchema(*options, clusterConfig); } break; - case NYql::NConnector::NApi::MS_SQL_SERVER: + case NYql::EGenericDataSourceKind::MS_SQL_SERVER: break; - case NYql::NConnector::NApi::POSTGRESQL: { + case NYql::EGenericDataSourceKind::POSTGRESQL: { auto* options = request.mutable_data_source_instance()->mutable_pg_options(); SetSchema(*options, clusterConfig); } break; - case NYql::NConnector::NApi::ORACLE: { + case NYql::EGenericDataSourceKind::ORACLE: { auto* options = request.mutable_data_source_instance()->mutable_oracle_options(); SetOracleServiceName(*options, clusterConfig); } break; - case NYql::NConnector::NApi::LOGGING: { + case NYql::EGenericDataSourceKind::LOGGING: { auto* options = request.mutable_data_source_instance()->mutable_logging_options(); SetLoggingFolderId(*options, clusterConfig); } break; default: - ythrow yexception() << "Unexpected data source kind: '" << NYql::NConnector::NApi::EDataSourceKind_Name(dataSourceKind) + ythrow yexception() << "Unexpected data source kind: '" << NYql::EGenericDataSourceKind_Name(dataSourceKind) << "'"; } } diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_state.h b/ydb/library/yql/providers/generic/provider/yql_generic_state.h index c217db35ef..9a4a003e04 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_state.h +++ b/ydb/library/yql/providers/generic/provider/yql_generic_state.h @@ -21,7 +21,7 @@ namespace NYql { const TStructExprType* ItemType = nullptr; TVector<TString> ColumnOrder; NYql::NConnector::NApi::TSchema Schema; - NYql::NConnector::NApi::TDataSourceInstance DataSourceInstance; + NYql::TGenericDataSourceInstance DataSourceInstance; }; using TGetTableResult = std::pair<std::optional<const TTableMeta*>, std::optional<TIssue>>; diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_utils.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_utils.cpp index b2ec216b3b..9d88896b88 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_utils.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_utils.cpp @@ -6,12 +6,12 @@ namespace NYql { TString DumpGenericClusterConfig(const TGenericClusterConfig& clusterConfig) { TStringBuilder sb; sb << "name = " << clusterConfig.GetName() - << ", kind = " << NConnector::NApi::EDataSourceKind_Name(clusterConfig.GetKind()) + << ", kind = " << NYql::EGenericDataSourceKind_Name(clusterConfig.GetKind()) << ", database name = " << clusterConfig.GetDatabaseName() << ", database id = " << clusterConfig.GetDatabaseId() << ", endpoint = " << clusterConfig.GetEndpoint() << ", use tls = " << clusterConfig.GetUseSsl() - << ", protocol = " << NConnector::NApi::EProtocol_Name(clusterConfig.GetProtocol()); + << ", protocol = " << NYql::EGenericProtocol_Name(clusterConfig.GetProtocol()); for (const auto& [key, value] : clusterConfig.GetDataSourceOptions()) { sb << ", " << key << " = " << value; diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_utils.h b/ydb/library/yql/providers/generic/provider/yql_generic_utils.h index 9bf173b1bf..4370e4f5ec 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_utils.h +++ b/ydb/library/yql/providers/generic/provider/yql_generic_utils.h @@ -1,7 +1,7 @@ #pragma once #include <yql/essentials/providers/common/proto/gateways_config.pb.h> -#include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> +#include <yql/essentials/providers/common/proto/gateways_config.pb.h> namespace NYql { TString DumpGenericClusterConfig(const TGenericClusterConfig& clusterConfig); diff --git a/ydb/tests/fq/generic/analytics/ya.make b/ydb/tests/fq/generic/analytics/ya.make index a17d05b8ee..33f675520e 100644 --- a/ydb/tests/fq/generic/analytics/ya.make +++ b/ydb/tests/fq/generic/analytics/ya.make @@ -54,7 +54,7 @@ IF (OPENSOURCE) ENDIF() PEERDIR( - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/utils ydb/tests/fq/generic/utils library/python/testing/recipe diff --git a/ydb/tests/fq/generic/streaming/ya.make b/ydb/tests/fq/generic/streaming/ya.make index 2183f7213e..fddcf60053 100644 --- a/ydb/tests/fq/generic/streaming/ya.make +++ b/ydb/tests/fq/generic/streaming/ya.make @@ -56,7 +56,7 @@ DEPENDS( ) PEERDIR( - ydb/library/yql/providers/generic/connector/api/common + yql/essentials/providers/common/proto ydb/library/yql/providers/generic/connector/tests/utils ydb/tests/fq/generic/utils ydb/tests/tools/datastreams_helpers |