diff options
author | vitalyisaev <vitalyisaev@ydb.tech> | 2023-10-13 16:52:24 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@ydb.tech> | 2023-10-13 17:23:27 +0300 |
commit | 3a5947c0ad5a5fb752980d58c5e55903edc3b4e4 (patch) | |
tree | a34c8d4279a25e835ea9274a2702386face73257 | |
parent | 6dbb54906effccb3979c234774e1b4d2b3ce83cc (diff) | |
download | ydb-3a5947c0ad5a5fb752980d58c5e55903edc3b4e4.tar.gz |
YQ Connector: refactor external databases and database resolving
1. Поправлено именование некоторых типов, связанных с ресолвингом MDB кластеров.
2. Устранено лишнее преобразование, подразумевавшее парсинг строки `TString -> std::pair<TString, ui32>`
3. Устранен лишний `enum`, функциональность которого полностью закрывается имеющимся `NYql::EDatabaseType`
4. Юнит-тесты на имплементации `IMdbEndpointGenerator` перенесены к ним поближе.
51 files changed, 580 insertions, 247 deletions
diff --git a/.mapping.json b/.mapping.json index 3cb003223e..543f7df328 100644 --- a/.mapping.json +++ b/.mapping.json @@ -4186,6 +4186,11 @@ "ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt":"", "ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.txt":"", "ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt":"", + "ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.darwin-x86_64.txt":"", + "ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-aarch64.txt":"", + "ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-x86_64.txt":"", + "ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.txt":"", + "ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.windows-x86_64.txt":"", "ydb/core/fq/libs/db_schema/CMakeLists.darwin-x86_64.txt":"", "ydb/core/fq/libs/db_schema/CMakeLists.linux-aarch64.txt":"", "ydb/core/fq/libs/db_schema/CMakeLists.linux-x86_64.txt":"", diff --git a/ydb/core/external_sources/CMakeLists.darwin-x86_64.txt b/ydb/core/external_sources/CMakeLists.darwin-x86_64.txt index da2b057478..69d796b77b 100644 --- a/ydb/core/external_sources/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/external_sources/CMakeLists.darwin-x86_64.txt @@ -15,7 +15,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC library-cpp-scheme ydb-core-base ydb-core-protos - generic-connector-libcpp + providers-common-db_id_async_resolver providers-s3-path_generator cpp-client-ydb_params cpp-client-ydb_value diff --git a/ydb/core/external_sources/CMakeLists.linux-aarch64.txt b/ydb/core/external_sources/CMakeLists.linux-aarch64.txt index ba50894c04..1b66f42a85 100644 --- a/ydb/core/external_sources/CMakeLists.linux-aarch64.txt +++ b/ydb/core/external_sources/CMakeLists.linux-aarch64.txt @@ -16,7 +16,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC library-cpp-scheme ydb-core-base ydb-core-protos - generic-connector-libcpp + providers-common-db_id_async_resolver providers-s3-path_generator cpp-client-ydb_params cpp-client-ydb_value diff --git a/ydb/core/external_sources/CMakeLists.linux-x86_64.txt b/ydb/core/external_sources/CMakeLists.linux-x86_64.txt index ba50894c04..1b66f42a85 100644 --- a/ydb/core/external_sources/CMakeLists.linux-x86_64.txt +++ b/ydb/core/external_sources/CMakeLists.linux-x86_64.txt @@ -16,7 +16,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC library-cpp-scheme ydb-core-base ydb-core-protos - generic-connector-libcpp + providers-common-db_id_async_resolver providers-s3-path_generator cpp-client-ydb_params cpp-client-ydb_value diff --git a/ydb/core/external_sources/CMakeLists.windows-x86_64.txt b/ydb/core/external_sources/CMakeLists.windows-x86_64.txt index da2b057478..69d796b77b 100644 --- a/ydb/core/external_sources/CMakeLists.windows-x86_64.txt +++ b/ydb/core/external_sources/CMakeLists.windows-x86_64.txt @@ -15,7 +15,7 @@ target_link_libraries(ydb-core-external_sources PUBLIC library-cpp-scheme ydb-core-base ydb-core-protos - generic-connector-libcpp + providers-common-db_id_async_resolver providers-s3-path_generator cpp-client-ydb_params cpp-client-ydb_value diff --git a/ydb/core/external_sources/external_source_factory.cpp b/ydb/core/external_sources/external_source_factory.cpp index a54602602b..d82157b5b1 100644 --- a/ydb/core/external_sources/external_source_factory.cpp +++ b/ydb/core/external_sources/external_source_factory.cpp @@ -6,7 +6,7 @@ #include <util/string/cast.h> #include <ydb/library/yql/providers/common/provider/yql_provider_names.h> -#include <ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h> +#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> namespace NKikimr::NExternalSource { @@ -34,13 +34,16 @@ private: IExternalSourceFactory::TPtr CreateExternalSourceFactory() { return MakeIntrusive<TExternalSourceFactory>(TMap<TString, IExternalSource::TPtr>{ - {"ObjectStorage", CreateObjectStorageExternalSource()}, { - ToString(NYql::NConnector::EExternalDataSource::ClickHouse), + ToString(NYql::EDatabaseType::ObjectStorage), + CreateObjectStorageExternalSource() + }, + { + ToString(NYql::EDatabaseType::ClickHouse), CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls"}) }, { - ToString(NYql::NConnector::EExternalDataSource::PostgreSQL), + ToString(NYql::EDatabaseType::PostgreSQL), CreateExternalDataSource(TString{NYql::GenericProviderName}, {"MDB_BASIC", "BASIC"}, {"database_name", "protocol", "mdb_cluster_id", "use_tls", "schema"}) } }); diff --git a/ydb/core/external_sources/ya.make b/ydb/core/external_sources/ya.make index c997a041f5..8a49100d7a 100644 --- a/ydb/core/external_sources/ya.make +++ b/ydb/core/external_sources/ya.make @@ -10,7 +10,7 @@ PEERDIR( library/cpp/scheme ydb/core/base ydb/core/protos - ydb/library/yql/providers/generic/connector/libcpp + ydb/library/yql/providers/common/db_id_async_resolver ydb/library/yql/providers/s3/path_generator ydb/public/sdk/cpp/client/ydb_params ydb/public/sdk/cpp/client/ydb_value diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp index b9c926a788..578ba3482e 100644 --- a/ydb/core/fq/libs/actors/database_resolver.cpp +++ b/ydb/core/fq/libs/actors/database_resolver.cpp @@ -19,18 +19,21 @@ namespace NFq { using namespace NActors; using namespace NYql; -using TEndpoint = NYql::TDatabaseResolverResponse::TEndpoint; -using TParser = std::function<TEndpoint(NJson::TJsonValue& body, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls)>; +using TDatabaseDescription = NYql::TDatabaseResolverResponse::TDatabaseDescription; +using TParser = std::function<TDatabaseDescription(NJson::TJsonValue& body, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls)>; using TParsers = THashMap<NYql::EDatabaseType, TParser>; -struct TDatabaseDescription { - TString Id; - NYql::EDatabaseType Type = NYql::EDatabaseType::Ydb; - NYql::TDatabaseAuth Auth; +struct TResolveParams { + // Treat ID as: + // - cluster ID (ClickHouse, PostgreSQL) + // - database ID (YDB) + TString Id; + NYql::EDatabaseType DatabaseType = NYql::EDatabaseType::Ydb; + NYql::TDatabaseAuth DatabaseAuth; }; -using TCache = TTtlCache<std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>, std::variant<TEndpoint, TString>>; -using TRequestMap = THashMap<NHttp::THttpOutgoingRequestPtr, TDatabaseDescription>; +using TCache = TTtlCache<std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>, std::variant<TDatabaseDescription, TString>>; +using TRequestMap = THashMap<NHttp::THttpOutgoingRequestPtr, TResolveParams>; class TResponseProcessor : public TActorBootstrapped<TResponseProcessor> { @@ -42,7 +45,7 @@ public: TResponseProcessor( const TActorId sender, TCache& cache, - const TDatabaseResolverResponse::TDatabaseEndpointsMap& ready, + const TDatabaseResolverResponse::TDatabaseDescriptionMap& ready, const TRequestMap& requests, const TString& traceId, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, @@ -85,7 +88,7 @@ private: TString errorMsg = "Could not resolve database ids: "; bool firstUnresolvedDbId = true; for (const auto& [_, params]: Requests) { - if (const auto it = DatabaseId2Endpoint.find(std::make_pair(params.Id, params.Type)); it == DatabaseId2Endpoint.end()) { + if (const auto it = DatabaseId2Endpoint.find(std::make_pair(params.Id, params.DatabaseType)); it == DatabaseId2Endpoint.end()) { errorMsg += (firstUnresolvedDbId ? TString{""} : TString{", "}) + params.Id; if (firstUnresolvedDbId) firstUnresolvedDbId = false; @@ -113,7 +116,7 @@ private: { TString status; TString errorMessage; - TMaybe<TEndpoint> result; + TMaybe<TDatabaseDescription> result; auto requestIter = Requests.find(ev->Get()->Request); HandledIds++; @@ -121,39 +124,39 @@ private: NJson::TJsonReaderConfig jsonConfig; NJson::TJsonValue databaseInfo; - LOG_D("Got databaseId response " << ev->Get()->Response->Body); + LOG_D("Got database id response " << ev->Get()->Response->Body); if (requestIter == Requests.end()) { - errorMessage = "Unknown databaseId"; + errorMessage = "Unknown database id"; } else { const auto& params = requestIter->second; const bool parseJsonOk = NJson::ReadJsonTree(ev->Get()->Response->Body, &jsonConfig, &databaseInfo); TParsers::const_iterator parserIt; - if (parseJsonOk && (parserIt = Parsers.find(params.Type)) != Parsers.end()) { + if (parseJsonOk && (parserIt = Parsers.find(params.DatabaseType)) != Parsers.end()) { try { - auto res = parserIt->second(databaseInfo, MdbEndpointGenerator, params.Auth.UseTls); - LOG_D("Got db_id: " << params.Id - << ", db type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(params.Type) + auto res = parserIt->second(databaseInfo, MdbEndpointGenerator, params.DatabaseAuth.UseTls); + LOG_D("database id: " << params.Id + << ", database type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(params.DatabaseType) << ", endpoint: " << res.Endpoint << ", database: " << res.Database); - DatabaseId2Endpoint[std::make_pair(params.Id, params.Type)] = res; + DatabaseId2Endpoint[std::make_pair(params.Id, params.DatabaseType)] = res; result.ConstructInPlace(res); } catch (...) { errorMessage = TStringBuilder() << " Couldn't resolve " - << "databaseId: " << params.Id - << ", db type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(params.Type) << "\n" + << "database id: " << params.Id + << ", database type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(params.DatabaseType) << "\n" << CurrentExceptionMessage(); } } else { errorMessage = TStringBuilder() << "Unable to parse database information. " - << "Database Id: " << params.Id - << ", db type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(params.Type); + << "database id: " << params.Id + << ", database type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(params.DatabaseType); } } } else { errorMessage = ev->Get()->Error; const TString error = TStringBuilder() - << "Cannot resolve databaseId (status = " + ToString(status) + "). " + << "Cannot resolve database id (status = " + ToString(status) + "). " << "Response body from " << ev->Get()->Request->URL << ": " << (ev->Get()->Response ? ev->Get()->Response->Body : "empty"); if (!errorMessage.empty()) { errorMessage += '\n'; @@ -166,7 +169,7 @@ private: Success = false; } else { const auto& params = requestIter->second; - auto key = std::make_tuple(params.Id, params.Type, params.Auth); + auto key = std::make_tuple(params.Id, params.DatabaseType, params.DatabaseAuth); if (errorMessage) { Cache.Put(key, errorMessage); } else { @@ -187,7 +190,7 @@ private: const TRequestMap Requests; const TString TraceId; const NYql::IMdbEndpointGenerator::TPtr MdbEndpointGenerator; - TDatabaseResolverResponse::TDatabaseEndpointsMap DatabaseId2Endpoint; + TDatabaseResolverResponse::TDatabaseDescriptionMap DatabaseId2Endpoint; size_t HandledIds = 0; bool Success = true; const TParsers& Parsers; @@ -226,7 +229,7 @@ public: } Y_ENSURE(endpoint); - return TEndpoint{endpoint, database, secure}; + return TDatabaseDescription{endpoint, "", 0, database, secure}; }; Parsers[NYql::EDatabaseType::Ydb] = ydbParser; Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) @@ -240,29 +243,32 @@ public: return ret; }; Parsers[NYql::EDatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) { - TString endpoint; + NYql::IMdbEndpointGenerator::TEndpoint endpoint; TVector<TString> aliveHosts; + for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) { if (host["health"].GetString() == "ALIVE" && host["type"].GetString() == "CLICKHOUSE") { aliveHosts.push_back(host["name"].GetString()); } } - if (!aliveHosts.empty()) { - endpoint = mdbEndpointGenerator->ToEndpoint( - NYql::EDatabaseType::ClickHouse, - aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], - useTls - ); - } - if (!endpoint) { + + if (aliveHosts.empty()) { ythrow yexception() << "No ALIVE ClickHouse hosts found"; } - return TEndpoint{endpoint, "", useTls}; + + endpoint = mdbEndpointGenerator->ToEndpoint( + NYql::EDatabaseType::ClickHouse, + aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], + useTls + ); + + return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls}; }; Parsers[NYql::EDatabaseType::PostgreSQL] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) { - TString endpoint; + NYql::IMdbEndpointGenerator::TEndpoint endpoint; TVector<TString> aliveHosts; + for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) { // all host services must be alive bool alive = true; @@ -277,19 +283,18 @@ public: aliveHosts.push_back(host["name"].GetString()); } } - if (!aliveHosts.empty()) { - endpoint = mdbEndpointGenerator->ToEndpoint( - NYql::EDatabaseType::PostgreSQL, - aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], - useTls - ); - } - - if (!endpoint) { + + if (aliveHosts.empty()) { ythrow yexception() << "No ALIVE PostgreSQL hosts found"; } - return TEndpoint{endpoint, "", useTls}; + endpoint = mdbEndpointGenerator->ToEndpoint( + NYql::EDatabaseType::PostgreSQL, + aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], + useTls + ); + + return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls}; }; } @@ -303,7 +308,7 @@ private: void SendResponse( const NActors::TActorId& recipient, - TDatabaseResolverResponse::TDatabaseEndpointsMap&& ready, + TDatabaseResolverResponse::TDatabaseDescriptionMap&& ready, bool success = true, const TString& errorMessage = "") { @@ -318,12 +323,12 @@ private: void Handle(TEvents::TEvEndpointRequest::TPtr ev) { TraceId = ev->Get()->TraceId; - LOG_D("Start databaseId resolver for " << ev->Get()->DatabaseIds.size() << " ids"); - TRequestMap requests; // request, (dbId, type, info) - TDatabaseResolverResponse::TDatabaseEndpointsMap ready; + LOG_D("Start database id resolver for " << ev->Get()->DatabaseIds.size() << " ids"); + TRequestMap requests; + TDatabaseResolverResponse::TDatabaseDescriptionMap ready; for (const auto& [p, databaseAuth] : ev->Get()->DatabaseIds) { const auto& [databaseId, databaseType] = p; - TMaybe<std::variant<TEndpoint, TString>> cacheVal; + TMaybe<std::variant<TDatabaseDescription, TString>> cacheVal; auto key = std::make_tuple(databaseId, databaseType, databaseAuth); if (Cache.Get(key, &cacheVal)) { switch(cacheVal->index()) { @@ -351,7 +356,7 @@ private: } else if (IsIn({NYql::EDatabaseType::ClickHouse, NYql::EDatabaseType::PostgreSQL }, databaseType)) { YQL_ENSURE(ev->Get()->MdbGateway, "empty MDB Gateway"); url = TUrlBuilder( - ev->Get()->MdbGateway + "/managed-" + NYql::DatabaseTypeToString(databaseType) + "/v1/clusters/") + ev->Get()->MdbGateway + "/managed-" + NYql::DatabaseTypeToMdbUrlPath(databaseType) + "/v1/clusters/") .AddPathComponent(databaseId) .AddPathComponent("hosts") .Build(); @@ -366,9 +371,9 @@ private: httpRequest->Set("Authorization", token); } - requests[httpRequest] = TDatabaseDescription{databaseId, databaseType, databaseAuth}; + requests[httpRequest] = TResolveParams{databaseId, databaseType, databaseAuth}; } catch (const std::exception& e) { - const TString msg = TStringBuilder() << " Error while preparing to resolve databaseId: " << databaseId << ", details: " << e.what(); + const TString msg = TStringBuilder() << " Error while preparing to resolve database id: " << databaseId << ", details: " << e.what(); LOG_E(msg); Cache.Put(key, msg); SendResponse(ev->Sender, {}, /*success=*/false, msg); diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt index 20c89d07b0..d9e0b7279f 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(fq-libs-db_id_async_resolver_impl) target_compile_options(fq-libs-db_id_async_resolver_impl PRIVATE diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt index 5dadab7c25..22a48ac0e7 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(fq-libs-db_id_async_resolver_impl) target_compile_options(fq-libs-db_id_async_resolver_impl PRIVATE diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt index 5dadab7c25..22a48ac0e7 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(fq-libs-db_id_async_resolver_impl) target_compile_options(fq-libs-db_id_async_resolver_impl PRIVATE diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt index 20c89d07b0..d9e0b7279f 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(ut) add_library(fq-libs-db_id_async_resolver_impl) target_compile_options(fq-libs-db_id_async_resolver_impl PRIVATE diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp index 742c3a07a9..c1c741f357 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp @@ -6,21 +6,23 @@ namespace NFq { return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net"; } + using TEndpoint = NYql::IMdbEndpointGenerator::TEndpoint; + // Currently we're going to use only HTTP protocol for ClickHouse - constexpr TStringBuf CLICKHOUSE_SECURE_PORT = "8443"; - constexpr TStringBuf CLICKHOUSE_INSECURE_PORT = "8123"; + constexpr ui32 CLICKHOUSE_SECURE_PORT = 8443; + constexpr ui32 CLICKHOUSE_INSECURE_PORT = 8123; // Managed PostgreSQL provides the only port both for secure and insecure connections - constexpr TStringBuf POSTGRESQL_PORT = "6432"; + constexpr ui32 POSTGRESQL_PORT = 6432; // TMdbEndpointGeneratorLegacy implements behavior required by YQL legacy ClickHouse provider class TMdbEndpointGeneratorLegacy: public NYql::IMdbEndpointGenerator { - TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override { + TEndpoint ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override { // Inherited from here // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r11819335#L27 if (databaseType == NYql::EDatabaseType::ClickHouse) { auto port = useTls ? CLICKHOUSE_SECURE_PORT : CLICKHOUSE_INSECURE_PORT; - return ReplaceDomain(mdbHost) + ":" + port; + return TEndpoint(ReplaceDomain(mdbHost), port); } ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); @@ -41,7 +43,7 @@ namespace NFq { { } - TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override { + TEndpoint ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override { auto fixedHost = TransformHost ? ReplaceDomain(mdbHost) : mdbHost; switch (databaseType) { @@ -49,11 +51,11 @@ namespace NFq { // https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect // TODO: fix Native protocol + TLS https://st.yandex-team.ru/YQ-2286 auto port = useTls ? CLICKHOUSE_SECURE_PORT : CLICKHOUSE_INSECURE_PORT; - return fixedHost + ":" + port; + return TEndpoint(fixedHost, port); } case NYql::EDatabaseType::PostgreSQL: // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect - return fixedHost + ":" + POSTGRESQL_PORT; + return TEndpoint(fixedHost, POSTGRESQL_PORT); default: ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); }; @@ -70,8 +72,8 @@ namespace NFq { // TMdbEndpointGeneratorNoop just does nothing class TMdbEndpointGeneratorNoop: public NYql::IMdbEndpointGenerator { - TString ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost, bool) const override { - return mdbHost; + TEndpoint ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost, bool) const override { + return TEndpoint(mdbHost, 0); } }; diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.darwin-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..d63f58a76d --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,67 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-fq-libs-db_id_async_resolver_impl-ut) +target_compile_options(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(ydb-core-fq-libs-db_id_async_resolver_impl-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + fq-libs-db_id_async_resolver_impl +) +target_link_options(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp +) +set_property( + TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-core-fq-libs-db_id_async_resolver_impl-ut + system_allocator +) +vcs_info(ydb-core-fq-libs-db_id_async_resolver_impl-ut) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-aarch64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..8e772cb53d --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-aarch64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-fq-libs-db_id_async_resolver_impl-ut) +target_compile_options(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(ydb-core-fq-libs-db_id_async_resolver_impl-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + fq-libs-db_id_async_resolver_impl +) +target_link_options(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp +) +set_property( + TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-core-fq-libs-db_id_async_resolver_impl-ut + cpp-malloc-jemalloc +) +vcs_info(ydb-core-fq-libs-db_id_async_resolver_impl-ut) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..c8af876b14 --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.linux-x86_64.txt @@ -0,0 +1,72 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-fq-libs-db_id_async_resolver_impl-ut) +target_compile_options(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(ydb-core-fq-libs-db_id_async_resolver_impl-ut PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + fq-libs-db_id_async_resolver_impl +) +target_link_options(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp +) +set_property( + TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-core-fq-libs-db_id_async_resolver_impl-ut + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(ydb-core-fq-libs-db_id_async_resolver_impl-ut) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.windows-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..b4c8d5f621 --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/CMakeLists.windows-x86_64.txt @@ -0,0 +1,60 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(ydb-core-fq-libs-db_id_async_resolver_impl-ut) +target_compile_options(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(ydb-core-fq-libs-db_id_async_resolver_impl-ut PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + fq-libs-db_id_async_resolver_impl +) +target_sources(ydb-core-fq-libs-db_id_async_resolver_impl-ut PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp +) +set_property( + TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_TARGET + ydb-core-fq-libs-db_id_async_resolver_impl-ut + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + ydb-core-fq-libs-db_id_async_resolver_impl-ut + PROPERTY + PROCESSORS + 1 +) +target_allocator(ydb-core-fq-libs-db_id_async_resolver_impl-ut + system_allocator +) +vcs_info(ydb-core-fq-libs-db_id_async_resolver_impl-ut) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp new file mode 100644 index 0000000000..3bd862a7e5 --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp @@ -0,0 +1,42 @@ +#include <library/cpp/testing/unittest/registar.h> +#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h> + +Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { + using TEndpoint = NYql::IMdbEndpointGenerator::TEndpoint; + + Y_UNIT_TEST(Legacy) { + auto transformer = NFq::MakeMdbEndpointGeneratorLegacy(); + + UNIT_ASSERT_VALUES_EQUAL( + transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1c-p5waby2y5y1kb5ue.db.yandex.net", true), + TEndpoint("rc1c-p5waby2y5y1kb5ue.db.yandex.net", 8443)); + + UNIT_ASSERT_VALUES_EQUAL( + transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "ya.ru", false), + TEndpoint("ya.db.yandex.net", 8123)); + } + + Y_UNIT_TEST(Generic_NoTransformHost) { + auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(false); + + UNIT_ASSERT_VALUES_EQUAL( + transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", true), + TEndpoint("rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", 8443)); + + UNIT_ASSERT_VALUES_EQUAL( + transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", false), + TEndpoint("rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", 6432)); + } + + Y_UNIT_TEST(Generic_WithTransformHost) { + auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(true); + + UNIT_ASSERT_VALUES_EQUAL( + transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", false), + TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 8123)); + + UNIT_ASSERT_VALUES_EQUAL( + transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", true), + TEndpoint("rc1b-eyt6dtobu96rwydq.db.yandex.net", 6432)); + } +} diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/ya.make b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/ya.make new file mode 100644 index 0000000000..bf8a4ffd2c --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/ya.make @@ -0,0 +1,15 @@ +UNITTEST() + +SIZE(SMALL) + +SRCS( + mdb_endpoint_generator_ut.cpp +) + +PEERDIR( + ydb/core/fq/libs/db_id_async_resolver_impl +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make index 49a3657e8f..97b9ea6f27 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make @@ -15,3 +15,8 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() + +RECURSE_FOR_TESTS( + ut +) + diff --git a/ydb/core/fq/libs/test_connection/test_data_streams.cpp b/ydb/core/fq/libs/test_connection/test_data_streams.cpp index 8e9fc1f02c..f5ae1ff9e3 100644 --- a/ydb/core/fq/libs/test_connection/test_data_streams.cpp +++ b/ydb/core/fq/libs/test_connection/test_data_streams.cpp @@ -165,8 +165,8 @@ private: return; } - auto it = response.DatabaseId2Endpoint.find(std::pair{ClusterConfig.GetDatabaseId(), NYql::EDatabaseType::DataStreams}); - if (it == response.DatabaseId2Endpoint.end()) { + auto it = response.DatabaseDescriptionMap.find(std::pair{ClusterConfig.GetDatabaseId(), NYql::EDatabaseType::DataStreams}); + if (it == response.DatabaseDescriptionMap.end()) { TC_LOG_E(Scope << " " << User << " " << NKikimr::MaskTicket(Token) << " Test data streams connection: database is not found for database_id " << ClusterConfig.GetDatabaseId()); ReplyError(TStringBuilder{} << "Test data streams connection: database is not found for database_id " << ClusterConfig.GetDatabaseId()); return; diff --git a/ydb/core/fq/libs/ya.make b/ydb/core/fq/libs/ya.make index 5327fd0e45..969a81fff3 100644 --- a/ydb/core/fq/libs/ya.make +++ b/ydb/core/fq/libs/ya.make @@ -11,6 +11,7 @@ RECURSE( control_plane_config control_plane_proxy control_plane_storage + db_id_async_resolver_impl db_schema events gateway diff --git a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_ut.cpp b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_ut.cpp index 68959b4bfa..910194df9f 100644 --- a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_ut.cpp @@ -2,7 +2,6 @@ #include <ydb/core/kqp/ut/federated_query/common/common.h> #include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h> #include <ydb/library/yql/providers/generic/connector/libcpp/client.h> -#include <ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h> #include <ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h> #include <ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/database_resolver_mock.h> #include <ydb/public/sdk/cpp/client/ydb_operation/operation.h> diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp index 5b05314fd7..b4d3d9e119 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_io_discovery.cpp @@ -15,7 +15,7 @@ using namespace NNodes; class TClickHouseIODiscoveryTransformer : public TGraphTransformerBase { -using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TEndpoint>; +using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TDatabaseDescription>; public: TClickHouseIODiscoveryTransformer(TClickHouseState::TPtr state) @@ -83,7 +83,7 @@ public: ctx.IssueManager.AddIssues(DbResolverResponse_->Issues); return TStatus::Error; } - FullResolvedIds_.insert(DbResolverResponse_->DatabaseId2Endpoint.begin(), DbResolverResponse_->DatabaseId2Endpoint.end()); + FullResolvedIds_.insert(DbResolverResponse_->DatabaseDescriptionMap.begin(), DbResolverResponse_->DatabaseDescriptionMap.end()); DbResolverResponse_ = std::make_shared<NYql::TDatabaseResolverResponse>(); YQL_CLOG(DEBUG, ProviderClickHouse) << "ResolvedIds: " << FullResolvedIds_.size(); auto& endpoints = State_->Configuration->Endpoints; diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.darwin-x86_64.txt index df50b83d34..2a1c24b27b 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.darwin-x86_64.txt @@ -6,12 +6,27 @@ # original buildsystem will not be accepted. +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) -add_library(providers-common-db_id_async_resolver INTERFACE) -target_link_libraries(providers-common-db_id_async_resolver INTERFACE +add_library(providers-common-db_id_async_resolver) +target_compile_options(providers-common-db_id_async_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-common-db_id_async_resolver PUBLIC contrib-libs-cxxsupp yutil cpp-threading-future connector-api-common yql-public-issue + tools-enum_parser-enum_serialization_runtime +) +generate_enum_serilization(providers-common-db_id_async_resolver + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h + INCLUDE_HEADERS + ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h ) diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-aarch64.txt index 6578ba76e0..c1487ce767 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-aarch64.txt @@ -6,13 +6,28 @@ # original buildsystem will not be accepted. +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) -add_library(providers-common-db_id_async_resolver INTERFACE) -target_link_libraries(providers-common-db_id_async_resolver INTERFACE +add_library(providers-common-db_id_async_resolver) +target_compile_options(providers-common-db_id_async_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-common-db_id_async_resolver PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil cpp-threading-future connector-api-common yql-public-issue + tools-enum_parser-enum_serialization_runtime +) +generate_enum_serilization(providers-common-db_id_async_resolver + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h + INCLUDE_HEADERS + ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h ) diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-x86_64.txt index 6578ba76e0..c1487ce767 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.linux-x86_64.txt @@ -6,13 +6,28 @@ # original buildsystem will not be accepted. +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) -add_library(providers-common-db_id_async_resolver INTERFACE) -target_link_libraries(providers-common-db_id_async_resolver INTERFACE +add_library(providers-common-db_id_async_resolver) +target_compile_options(providers-common-db_id_async_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-common-db_id_async_resolver PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil cpp-threading-future connector-api-common yql-public-issue + tools-enum_parser-enum_serialization_runtime +) +generate_enum_serilization(providers-common-db_id_async_resolver + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h + INCLUDE_HEADERS + ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h ) diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.windows-x86_64.txt index df50b83d34..2a1c24b27b 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/common/db_id_async_resolver/CMakeLists.windows-x86_64.txt @@ -6,12 +6,27 @@ # original buildsystem will not be accepted. +get_built_tool_path( + TOOL_enum_parser_bin + TOOL_enum_parser_dependency + tools/enum_parser/enum_parser + enum_parser +) -add_library(providers-common-db_id_async_resolver INTERFACE) -target_link_libraries(providers-common-db_id_async_resolver INTERFACE +add_library(providers-common-db_id_async_resolver) +target_compile_options(providers-common-db_id_async_resolver PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(providers-common-db_id_async_resolver PUBLIC contrib-libs-cxxsupp yutil cpp-threading-future connector-api-common yql-public-issue + tools-enum_parser-enum_serialization_runtime +) +generate_enum_serilization(providers-common-db_id_async_resolver + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h + INCLUDE_HEADERS + ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h ) diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h index 142ba04d9f..e39451f570 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h +++ b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h @@ -2,6 +2,7 @@ #include <library/cpp/threading/future/future.h> #include <util/string/builder.h> +#include <util/string/cast.h> #include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> #include <ydb/library/yql/public/issue/yql_issue.h> @@ -15,25 +16,39 @@ enum class EDatabaseType { PostgreSQL }; -inline EDatabaseType DataSourceKindToDatabaseType(NConnector::NApi::EDataSourceKind dataSourceKind) { +inline EDatabaseType DatabaseTypeFromDataSourceKind(NConnector::NApi::EDataSourceKind dataSourceKind) { switch (dataSourceKind) { case NConnector::NApi::EDataSourceKind::POSTGRESQL: return EDatabaseType::PostgreSQL; case NConnector::NApi::EDataSourceKind::CLICKHOUSE: return EDatabaseType::ClickHouse; default: - ythrow yexception() << TStringBuf() << "Unknown data source kind: " << NConnector::NApi::EDataSourceKind_Name(dataSourceKind); + ythrow yexception() << "Unknown data source kind: " << NConnector::NApi::EDataSourceKind_Name(dataSourceKind); } } -inline TString DatabaseTypeToString(EDatabaseType databaseType) { +inline NConnector::NApi::EDataSourceKind DatabaseTypeToDataSourceKind(EDatabaseType databaseType) { + switch (databaseType) { + case EDatabaseType::PostgreSQL: + return NConnector::NApi::EDataSourceKind::POSTGRESQL; + case EDatabaseType::ClickHouse: + return NConnector::NApi::EDataSourceKind::CLICKHOUSE; + default: + ythrow yexception() << "Unknown database type: " << ToString(databaseType); + } +} + +inline TString DatabaseTypeToMdbUrlPath(EDatabaseType databaseType) { + auto dump = ToString(databaseType); + dump.to_lower(); + switch (databaseType) { case EDatabaseType::ClickHouse: - return "clickhouse"; + return dump; case EDatabaseType::PostgreSQL: - return "postgresql"; + return dump; default: - ythrow yexception() << TStringBuf() << "Unknown database type: " << int(databaseType); + ythrow yexception() << "Unsupported database type: " << ToString(databaseType); } } @@ -59,37 +74,28 @@ struct TDatabaseAuth { struct TDatabaseResolverResponse { - struct TEndpoint { - std::tuple<TString, ui32> ParseHostPort() const { - size_t pos = Endpoint.find(':'); - if (pos == TString::npos) { - ythrow yexception() << TStringBuilder() << "Endpoint '" << Endpoint << "' contains no ':' separator"; - } - - auto host = Endpoint.substr(0, pos); - auto port = static_cast<ui32>(std::stoi(Endpoint.substr(pos + 1))); - - return std::make_tuple(std::move(host), port); - } - + struct TDatabaseDescription { TString Endpoint; + TString Host; + ui32 Port = 0; TString Database; bool Secure = false; }; - using TDatabaseEndpointsMap = THashMap<std::pair<TString, EDatabaseType>, TEndpoint>; + // key - (database id, database type), value - resolved database params + using TDatabaseDescriptionMap = THashMap<std::pair<TString, EDatabaseType>, TDatabaseDescription>; TDatabaseResolverResponse() = default; TDatabaseResolverResponse( - TDatabaseEndpointsMap&& databaseId2Endpoint, + TDatabaseDescriptionMap&& databaseDescriptionMap, bool success = false, const NYql::TIssues& issues = {}) - : DatabaseId2Endpoint(std::move(databaseId2Endpoint)) + : DatabaseDescriptionMap(std::move(databaseDescriptionMap)) , Success(success) , Issues(issues) {} - TDatabaseEndpointsMap DatabaseId2Endpoint; + TDatabaseDescriptionMap DatabaseDescriptionMap; bool Success = false; NYql::TIssues Issues; }; diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h index 2674ffb16a..a5518bbdbe 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h +++ b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h @@ -10,11 +10,14 @@ namespace NYql { class IMdbEndpointGenerator { public: using TPtr = std::shared_ptr<IMdbEndpointGenerator>; + using TEndpoint = std::pair<TString, ui32>; - virtual TString ToEndpoint( + // Returns FQDN and port of the database host + virtual TEndpoint ToEndpoint( const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const = 0; + virtual ~IMdbEndpointGenerator() = default; }; } diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make index f20098f9b2..255eb1931d 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make +++ b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make @@ -13,4 +13,6 @@ PEERDIR( YQL_LAST_ABI_VERSION() +GENERATE_ENUM_SERIALIZATION(db_async_resolver.h) + END() diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt index 82847e3680..7f98ef6355 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt @@ -8,12 +8,6 @@ add_subdirectory(cli) add_subdirectory(ut_helpers) -get_built_tool_path( - TOOL_enum_parser_bin - TOOL_enum_parser_dependency - tools/enum_parser/enum_parser - enum_parser -) add_library(generic-connector-libcpp) target_link_libraries(generic-connector-libcpp PUBLIC @@ -29,7 +23,6 @@ target_link_libraries(generic-connector-libcpp PUBLIC connector-api-service yql-public-issue library-yql-utils - tools-enum_parser-enum_serialization_runtime ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp @@ -37,8 +30,3 @@ target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/error.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/utils.cpp ) -generate_enum_serilization(generic-connector-libcpp - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h - INCLUDE_HEADERS - ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h -) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt index 822344f764..ec7f58bac4 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt @@ -8,12 +8,6 @@ add_subdirectory(cli) add_subdirectory(ut_helpers) -get_built_tool_path( - TOOL_enum_parser_bin - TOOL_enum_parser_dependency - tools/enum_parser/enum_parser - enum_parser -) add_library(generic-connector-libcpp) target_link_libraries(generic-connector-libcpp PUBLIC @@ -30,7 +24,6 @@ target_link_libraries(generic-connector-libcpp PUBLIC connector-api-service yql-public-issue library-yql-utils - tools-enum_parser-enum_serialization_runtime ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp @@ -38,8 +31,3 @@ target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/error.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/utils.cpp ) -generate_enum_serilization(generic-connector-libcpp - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h - INCLUDE_HEADERS - ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h -) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt index 822344f764..ec7f58bac4 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt @@ -8,12 +8,6 @@ add_subdirectory(cli) add_subdirectory(ut_helpers) -get_built_tool_path( - TOOL_enum_parser_bin - TOOL_enum_parser_dependency - tools/enum_parser/enum_parser - enum_parser -) add_library(generic-connector-libcpp) target_link_libraries(generic-connector-libcpp PUBLIC @@ -30,7 +24,6 @@ target_link_libraries(generic-connector-libcpp PUBLIC connector-api-service yql-public-issue library-yql-utils - tools-enum_parser-enum_serialization_runtime ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp @@ -38,8 +31,3 @@ target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/error.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/utils.cpp ) -generate_enum_serilization(generic-connector-libcpp - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h - INCLUDE_HEADERS - ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h -) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt index 82847e3680..7f98ef6355 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt @@ -8,12 +8,6 @@ add_subdirectory(cli) add_subdirectory(ut_helpers) -get_built_tool_path( - TOOL_enum_parser_bin - TOOL_enum_parser_dependency - tools/enum_parser/enum_parser - enum_parser -) add_library(generic-connector-libcpp) target_link_libraries(generic-connector-libcpp PUBLIC @@ -29,7 +23,6 @@ target_link_libraries(generic-connector-libcpp PUBLIC connector-api-service yql-public-issue library-yql-utils - tools-enum_parser-enum_serialization_runtime ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp @@ -37,8 +30,3 @@ target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/error.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/utils.cpp ) -generate_enum_serilization(generic-connector-libcpp - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h - INCLUDE_HEADERS - ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h -) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h b/ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h deleted file mode 100644 index 1e2c3b2f44..0000000000 --- a/ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -namespace NYql::NConnector { - // List of external datasources supported by Connector. - // Serialized values from this enum are used within - // `CREATE EXTERNAL DATA SOURCE` syntax. - enum class EExternalDataSource { - ClickHouse, - PostgreSQL - }; -} diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp index 2fa4b253a0..d7cab1ebd0 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.cpp @@ -53,8 +53,7 @@ namespace NYql::NConnector::NTest { const TString& password, bool useTls, const TString& serviceAccountId, - const TString& serviceAccountIdSignature, - NYql::NConnector::EExternalDataSource sourceType) + const TString& serviceAccountIdSignature) { auto tc = kikimr->GetTableClient(); auto session = tc.CreateSession().GetValueSync().GetSession(); @@ -83,7 +82,7 @@ namespace NYql::NConnector::NTest { "protocol"_a = NYql::NConnector::NApi::EProtocol_Name(protocol), "service_account_id"_a = serviceAccountId, "service_account_id_signature"_a = serviceAccountIdSignature, - "source_type"_a = ToString(sourceType)); + "source_type"_a = ToString(NYql::EDatabaseType::ClickHouse)); auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); } diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h index 58fb24caf4..bed38da101 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h @@ -160,8 +160,7 @@ namespace NYql::NConnector::NTest { const TString& password = DEFAULT_PASSWORD, bool useTls = DEFAULT_USE_TLS, const TString& serviceAccountId = DEFAULT_CH_SERVICE_ACCOUNT_ID, - const TString& serviceAccountIdSignature = DEFAULT_CH_SERVICE_ACCOUNT_ID_SIGNATURE, - NYql::NConnector::EExternalDataSource sourceType = DEFAULT_CH_SOURCE_TYPE); + const TString& serviceAccountIdSignature = DEFAULT_CH_SERVICE_ACCOUNT_ID_SIGNATURE); class TConnectorClientMock: public NYql::NConnector::IClient { public: diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/database_resolver_mock.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/database_resolver_mock.h index b21259cdcd..567b691209 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/database_resolver_mock.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/database_resolver_mock.h @@ -20,7 +20,8 @@ namespace NYql::NConnector::NTest { void AddClickHouseCluster( const TString& clusterId = DEFAULT_CH_CLUSTER_ID, - const TString& endpoint = DEFAULT_CH_ENDPOINT, + const TString& hostFqdn = DEFAULT_CH_HOST, + const ui32 port = DEFAULT_CH_PORT, const TString& serviceAccountId = DEFAULT_CH_SERVICE_ACCOUNT_ID, const TString& serviceAccountIdSignature = DEFAULT_CH_SERVICE_ACCOUNT_ID_SIGNATURE) { NYql::IDatabaseAsyncResolver::TDatabaseAuthMap dbResolverReq; @@ -30,11 +31,11 @@ namespace NYql::NConnector::NTest { true, true}; - NYql::TDatabaseResolverResponse::TDatabaseEndpointsMap databaseEndpointsMap; - databaseEndpointsMap[std::make_pair(clusterId, NYql::EDatabaseType::ClickHouse)] = - NYql::TDatabaseResolverResponse::TEndpoint{endpoint, clusterId}; + NYql::TDatabaseResolverResponse::TDatabaseDescriptionMap databaseDescriptions; + databaseDescriptions[std::make_pair(clusterId, NYql::EDatabaseType::ClickHouse)] = + NYql::TDatabaseResolverResponse::TDatabaseDescription{"", hostFqdn, port, clusterId}; auto dbResolverPromise = NThreading::NewPromise<NYql::TDatabaseResolverResponse>(); - dbResolverPromise.SetValue(NYql::TDatabaseResolverResponse(std::move(databaseEndpointsMap), true)); + dbResolverPromise.SetValue(NYql::TDatabaseResolverResponse(std::move(databaseDescriptions), true)); auto result = dbResolverPromise.GetFuture(); EXPECT_CALL(*this, ResolveIds(DatabaseAuthMapMatcher(dbResolverReq))) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h index e2b82727ed..96d5f48835 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/defaults.h @@ -1,6 +1,5 @@ #pragma once #include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> -#include <ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h> #include <util/generic/string.h> #include <util/string/builder.h> @@ -23,7 +22,6 @@ namespace NYql::NConnector::NTest { extern const TString DEFAULT_CH_ENDPOINT; extern const TString DEFAULT_CH_CLUSTER_ID; constexpr NApi::EProtocol DEFAULT_CH_PROTOCOL = NApi::EProtocol::HTTP; - constexpr NYql::NConnector::EExternalDataSource DEFAULT_CH_SOURCE_TYPE = NYql::NConnector::EExternalDataSource::ClickHouse; extern const TString DEFAULT_CH_SERVICE_ACCOUNT_ID; extern const TString DEFAULT_CH_SERVICE_ACCOUNT_ID_SIGNATURE; } // namespace NYql::NConnector::NTest diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ya.make b/ydb/library/yql/providers/generic/connector/libcpp/ya.make index f389d52c34..2adc1cf72a 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ya.make +++ b/ydb/library/yql/providers/generic/connector/libcpp/ya.make @@ -20,8 +20,6 @@ PEERDIR( ydb/library/yql/utils ) -GENERATE_ENUM_SERIALIZATION(external_data_source.h) - END() RECURSE( diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp index ae09598204..542cebc9f5 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_cluster_config.cpp @@ -6,7 +6,7 @@ #include <util/generic/yexception.h> #include <ydb/library/yql/providers/generic/connector/api/common/data_source.pb.h> -#include <ydb/library/yql/providers/generic/connector/libcpp/external_data_source.h> +#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> #include "yql_generic_cluster_config.h" @@ -145,11 +145,6 @@ namespace NYql { clusterConfig.SetDatabaseId(it->second); } - static THashMap<EExternalDataSource, EDataSourceKind> DataSourceApiMapping = { - {EExternalDataSource::ClickHouse, EDataSourceKind::CLICKHOUSE}, - {EExternalDataSource::PostgreSQL, EDataSourceKind::POSTGRESQL}, - }; - void ParseSourceType(const THashMap<TString, TString>& properties, NYql::TGenericClusterConfig& clusterConfig) { auto it = properties.find("source_type"); @@ -158,17 +153,7 @@ namespace NYql { ythrow yexception() << "missing 'SOURCE_TYPE' value"; } - EExternalDataSource externalDataSource; - if (!TryFromString<EExternalDataSource>(it->second, externalDataSource)) { - ythrow yexception() << "invalid 'SOURCE_TYPE' value: '" << it->second - << "', valid types are: " << GetEnumAllNames<EExternalDataSource>(); - } - - if (!DataSourceApiMapping.contains(externalDataSource)) { - ythrow yexception() << "cannot map 'SOURCE_TYPE' value: '" << it->second << "' into Connector API value"; - } - - clusterConfig.SetKind(DataSourceApiMapping.at(externalDataSource)); + clusterConfig.SetKind(DatabaseTypeToDataSourceKind(FromString<NYql::EDatabaseType>(it->second))); } void ParseProtocol(const THashMap<TString, TString>& properties, diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp index 83fbf8a405..a9dfc5d376 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_io_discovery.cpp @@ -53,7 +53,7 @@ namespace NYql { auto databaseId = cluster.GetDatabaseId(); if (databaseId) { YQL_CLOG(DEBUG, ProviderGeneric) << "found database id: " << databaseId; - const auto idKey = std::make_pair(databaseId, DataSourceKindToDatabaseType(cluster.GetKind())); + const auto idKey = std::make_pair(databaseId, DatabaseTypeFromDataSourceKind(cluster.GetKind())); const auto iter = State_->DatabaseAuth.find(idKey); if (iter != State_->DatabaseAuth.end()) { YQL_CLOG(DEBUG, ProviderGeneric) << "resolve database id: " << databaseId; @@ -93,7 +93,7 @@ namespace NYql { } // Copy resolver results and reallocate pointer - auto databaseIdsToEndpointsResolved = std::move(DbResolverResponse_->DatabaseId2Endpoint); + auto databaseIdsToEndpointsResolved = std::move(DbResolverResponse_->DatabaseDescriptionMap); DbResolverResponse_ = std::make_shared<NYql::TDatabaseResolverResponse>(); @@ -107,15 +107,18 @@ namespace NYql { } private: - TStatus ModifyClusterConfigs(const TDatabaseResolverResponse::TDatabaseEndpointsMap& databaseIdsToEndpoints, TExprContext& ctx) { + TStatus ModifyClusterConfigs(const TDatabaseResolverResponse::TDatabaseDescriptionMap& databaseDescriptions, TExprContext& ctx) { const auto& databaseIdsToClusterNames = State_->Configuration->DatabaseIdsToClusterNames; auto& clusterNamesToClusterConfigs = State_->Configuration->ClusterNamesToClusterConfigs; - for (const auto& [databaseIdWithType, endpointSrc] : databaseIdsToEndpoints) { + for (const auto& [databaseIdWithType, databaseDescription] : databaseDescriptions) { const auto& databaseId = databaseIdWithType.first; - YQL_CLOG(DEBUG, ProviderGeneric) << "resolved database id: " << databaseId - << ", endpoint: " << endpointSrc.Endpoint; + Y_ENSURE(databaseDescription.Host, "Empty resolved database host"); + Y_ENSURE(databaseDescription.Port, "Empty resolved database port"); + YQL_CLOG(INFO, ProviderGeneric) << "resolved database id: " << databaseId + << ", host: " << databaseDescription.Host + << ", port: " << databaseDescription.Port; auto clusterNamesIter = databaseIdsToClusterNames.find(databaseId); @@ -136,10 +139,9 @@ namespace NYql { return TStatus::Error; } - auto hostPort = endpointSrc.ParseHostPort(); auto endpointDst = clusterConfigIter->second.mutable_endpoint(); - endpointDst->set_host(std::get<0>(hostPort)); - endpointDst->set_port(std::get<1>(hostPort)); + endpointDst->set_host(databaseDescription.Host); + endpointDst->set_port(databaseDescription.Port); } } diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp index 6afb57a2ef..c050651f63 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp @@ -43,7 +43,7 @@ namespace NYql { const auto token = MakeStructuredToken(clusterConfig, credentials); - databaseAuth[std::make_pair(databaseId, DataSourceKindToDatabaseType(clusterConfig.GetKind()))] = + databaseAuth[std::make_pair(databaseId, DatabaseTypeFromDataSourceKind(clusterConfig.GetKind()))] = NYql::TDatabaseAuth{token, /*AddBearer=*/true, clusterConfig.GetUseSsl()}; DatabaseIdsToClusterNames[databaseId].emplace_back(clusterName); diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp index 9608c05602..abc54468ed 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasink_io_discovery.cpp @@ -12,7 +12,7 @@ namespace { using namespace NNodes; class TPqDataSinkIODiscoveryTransformer : public TGraphTransformerBase { -using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TEndpoint>; +using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TDatabaseDescription>; public: explicit TPqDataSinkIODiscoveryTransformer(TPqState::TPtr state) : State_(state) @@ -54,7 +54,7 @@ public: ctx.IssueManager.AddIssues(DbResolverResponse_->Issues); return TStatus::Error; } - FullResolvedIds_.insert(DbResolverResponse_->DatabaseId2Endpoint.begin(), DbResolverResponse_->DatabaseId2Endpoint.end()); + FullResolvedIds_.insert(DbResolverResponse_->DatabaseDescriptionMap.begin(), DbResolverResponse_->DatabaseDescriptionMap.end()); DbResolverResponse_ = std::make_shared<NYql::TDatabaseResolverResponse>(); FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_); return TStatus::Ok; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp index d845e40e90..c488e37221 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.cpp @@ -79,7 +79,7 @@ void FindYdsDbIdsForResolving( void FillSettingsWithResolvedYdsIds( const TPqState::TPtr& state, - const TDatabaseResolverResponse::TDatabaseEndpointsMap& fullResolvedIds) + const TDatabaseResolverResponse::TDatabaseDescriptionMap& fullResolvedIds) { YQL_CLOG(INFO, ProviderPq) << "FullResolvedIds size: " << fullResolvedIds.size(); auto& clusters = state->Configuration->ClustersConfigurationSettings; diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h index 8cd752b3bb..0512bb1427 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_helpers.h @@ -19,6 +19,6 @@ void FindYdsDbIdsForResolving( void FillSettingsWithResolvedYdsIds( const TPqState::TPtr& state, - const TDatabaseResolverResponse::TDatabaseEndpointsMap& fullResolvedIds); + const TDatabaseResolverResponse::TDatabaseDescriptionMap& fullResolvedIds); } // namespace NYql diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp index cc6939426e..a2bbda0bf9 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_io_discovery.cpp @@ -13,7 +13,7 @@ using namespace NNodes; class TPqIODiscoveryTransformer : public TGraphTransformerBase { -using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TEndpoint>; +using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TDatabaseDescription>; public: explicit TPqIODiscoveryTransformer(TPqState::TPtr state) @@ -56,7 +56,7 @@ public: ctx.IssueManager.AddIssues(DbResolverResponse_->Issues); return TStatus::Error; } - FullResolvedIds_.insert(DbResolverResponse_->DatabaseId2Endpoint.begin(), DbResolverResponse_->DatabaseId2Endpoint.end()); + FullResolvedIds_.insert(DbResolverResponse_->DatabaseDescriptionMap.begin(), DbResolverResponse_->DatabaseDescriptionMap.end()); DbResolverResponse_ = std::make_shared<NYql::TDatabaseResolverResponse>(); FillSettingsWithResolvedYdsIds(State_, FullResolvedIds_); return TStatus::Ok; diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp index 28ecf98338..0a2d05bb22 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_io_discovery.cpp @@ -17,7 +17,7 @@ namespace { using namespace NNodes; class TYdbIODiscoveryTransformer : public TGraphTransformerBase { -using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TEndpoint>; +using TDbId2Endpoint = THashMap<std::pair<TString, NYql::EDatabaseType>, NYql::TDatabaseResolverResponse::TDatabaseDescription>; public: TYdbIODiscoveryTransformer(TYdbState::TPtr state) : State_(std::move(state)) @@ -77,7 +77,7 @@ public: ctx.IssueManager.AddIssues(DbResolverResponse_->Issues); return TStatus::Error; } - FullResolvedIds_.insert(DbResolverResponse_->DatabaseId2Endpoint.begin(), DbResolverResponse_->DatabaseId2Endpoint.end()); + FullResolvedIds_.insert(DbResolverResponse_->DatabaseDescriptionMap.begin(), DbResolverResponse_->DatabaseDescriptionMap.end()); DbResolverResponse_ = std::make_shared<NYql::TDatabaseResolverResponse>(); auto& clusters = State_->Configuration->Clusters; const auto& id2Clusters = State_->Configuration->DbId2Clusters; diff --git a/ydb/services/fq/ut_integration/fq_ut.cpp b/ydb/services/fq/ut_integration/fq_ut.cpp index 4b2e8c9aac..1b5764fee4 100644 --- a/ydb/services/fq/ut_integration/fq_ut.cpp +++ b/ydb/services/fq/ut_integration/fq_ut.cpp @@ -9,7 +9,6 @@ #include <ydb/core/fq/libs/control_plane_storage/message_builders.h> #include <ydb/core/fq/libs/actors/database_resolver.h> -#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> @@ -857,33 +856,6 @@ Y_UNIT_TEST_SUITE(Yq_1) { } Y_UNIT_TEST_SUITE(Yq_2) { - // use fork for data test due to ch initialization problem - Y_UNIT_TEST(Test_HostNameTrasformation) { - { - auto transformer = ::NFq::MakeMdbEndpointGeneratorLegacy(); - UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1c-p5waby2y5y1kb5ue.db.yandex.net", true), - "rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443"); - UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "ya.ru", false), - "ya.db.yandex.net:8123"); - } - - { - auto transformer = ::NFq::MakeMdbEndpointGeneratorGeneric(false); - UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", true), - "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net:8443"); - UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", false), - "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net:6432"); - } - - { - auto transformer = ::NFq::MakeMdbEndpointGeneratorGeneric(true); - UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", false), - "rc1a-d6dv17lv47v5mcop.db.yandex.net:8123"); - UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", true), - "rc1b-eyt6dtobu96rwydq.db.yandex.net:6432"); - } - } - SIMPLE_UNIT_FORKED_TEST(ReadFromYdbOverYq) { TKikimrWithGrpcAndRootSchema server({}, {}, {}, true); ui16 grpc = server.GetPort(); |