diff options
author | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-08-24 21:21:48 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-08-24 21:59:13 +0300 |
commit | 24fce6d1ca9debced3e88aec3a404aa233a6c1c9 (patch) | |
tree | f6469161308affe21082411f8a4101cb32f72d45 | |
parent | 7a0f7576d01bee0c781e115b219d34c475d74e67 (diff) | |
download | ydb-24fce6d1ca9debced3e88aec3a404aa233a6c1c9.tar.gz |
YQ Connector: use MdbTransformHost config field again
В конфигурации YQ есть поле `MdbTransformHost`. Ранее оно было помечено как устаревшее, теперь снова используется.
19 files changed, 133 insertions, 110 deletions
diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp index ee5aaccd8cd..9591ff9b51d 100644 --- a/ydb/core/fq/libs/actors/database_resolver.cpp +++ b/ydb/core/fq/libs/actors/database_resolver.cpp @@ -20,7 +20,7 @@ using namespace NYql; using TEndpoint = NYql::TDatabaseResolverResponse::TEndpoint; -using TParser = std::function<TEndpoint(NJson::TJsonValue& body, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer)>; +using TParser = std::function<TEndpoint(NJson::TJsonValue& body, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator)>; using TParsers = THashMap<NYql::EDatabaseType, TParser>; using TCache = TTtlCache<std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>, std::variant<TEndpoint, TString>>; @@ -38,13 +38,13 @@ public: const TDatabaseResolverResponse::TDatabaseEndpointsMap& ready, const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>>& requests, const TString& traceId, - const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer, + const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, const TParsers& parsers) : Sender(sender) , Cache(cache) , Requests(requests) , TraceId(traceId) - , MdbHostTransformer(mdbHostTransformer) + , MdbEndpointGenerator(mdbEndpointGenerator) , DatabaseId2Endpoint(ready) , Parsers(parsers) { } @@ -127,7 +127,7 @@ private: TParsers::const_iterator parserIt; if (parseJsonOk && (parserIt = Parsers.find(databaseType)) != Parsers.end()) { try { - auto res = parserIt->second(databaseInfo, MdbHostTransformer); + auto res = parserIt->second(databaseInfo, MdbEndpointGenerator); LOG_D("Got db_id: " << databaseId << ", db type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(databaseType) << ", endpoint: " << res.Endpoint @@ -184,7 +184,7 @@ private: TCache& Cache; const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>> Requests; const TString TraceId; - const NYql::IMdbHostTransformer::TPtr MdbHostTransformer; + const NYql::IMdbEndpointGenerator::TPtr MdbEndpointGenerator; TDatabaseResolverResponse::TDatabaseEndpointsMap DatabaseId2Endpoint; size_t HandledIds = 0; bool Success = true; @@ -205,7 +205,7 @@ public: .SetErrorTtl(TDuration::Minutes(1)) .SetMaxSize(1000000)) { - auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr&) { + auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&) { bool secure = false; TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust(); TString prefix("/?database="); @@ -227,9 +227,9 @@ public: return TEndpoint{endpoint, database, secure}; }; Parsers[NYql::EDatabaseType::Ydb] = ydbParser; - Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer) + Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator) { - auto ret = ydbParser(databaseInfo, mdbHostTransformer); + auto ret = ydbParser(databaseInfo, mdbEndpointGenerator); // TODO: Take explicit field from MVP if (ret.Endpoint.StartsWith("ydb.")) { // Replace "ydb." -> "yds." @@ -237,7 +237,7 @@ public: } return ret; }; - Parsers[NYql::EDatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer) { + Parsers[NYql::EDatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator) { TString endpoint; TVector<TString> aliveHosts; for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) { @@ -246,7 +246,7 @@ public: } } if (!aliveHosts.empty()) { - endpoint = mdbHostTransformer->ToEndpoint( + endpoint = mdbEndpointGenerator->ToEndpoint( NYql::EDatabaseType::ClickHouse, aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())] ); @@ -257,7 +257,7 @@ public: return TEndpoint{endpoint, "", true}; }; - Parsers[NYql::EDatabaseType::PostgreSQL] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer) { + Parsers[NYql::EDatabaseType::PostgreSQL] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator) { TString endpoint; TVector<TString> aliveHosts; for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) { @@ -276,7 +276,7 @@ public: } } if (!aliveHosts.empty()) { - endpoint = mdbHostTransformer->ToEndpoint( + endpoint = mdbEndpointGenerator->ToEndpoint( NYql::EDatabaseType::PostgreSQL, aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())] ); @@ -375,7 +375,7 @@ private: if (!requests.empty()) { auto helper = Register( - new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbHostTransformer, Parsers)); + new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbEndpointGenerator, Parsers)); for (const auto& [request, _] : requests) { TActivationContext::Send(new IEventHandle(HttpProxy, helper, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(request))); diff --git a/ydb/core/fq/libs/actors/run_actor.cpp b/ydb/core/fq/libs/actors/run_actor.cpp index f2977c626ef..7fea6502a66 100644 --- a/ydb/core/fq/libs/actors/run_actor.cpp +++ b/ydb/core/fq/libs/actors/run_actor.cpp @@ -59,7 +59,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/core/fq/libs/control_plane_storage/util.h> #include <ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h> -#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h> +#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h> #include <ydb/core/fq/libs/gateway/empty_gateway.h> #include <ydb/core/fq/libs/private_client/events.h> #include <ydb/core/fq/libs/private_client/private_client.h> @@ -1832,8 +1832,7 @@ private: Params.DatabaseResolver, Params.Config.GetCommon().GetYdbMvpCloudEndpoint(), Params.Config.GetCommon().GetMdbGateway(), - NFq::MakeTMdbHostTransformerGeneric(), - // Params.Config.GetCommon().GetMdbTransformHost(), + NFq::MakeMdbEndpointGeneratorGeneric(Params.Config.GetCommon().GetMdbTransformHost()), Params.QueryId); { // TBD: move init to better place diff --git a/ydb/core/fq/libs/config/protos/common.proto b/ydb/core/fq/libs/config/protos/common.proto index ade8d5562c7..7bb6da3577b 100644 --- a/ydb/core/fq/libs/config/protos/common.proto +++ b/ydb/core/fq/libs/config/protos/common.proto @@ -17,7 +17,7 @@ message TCommonConfig { string YdbMvpCloudEndpoint = 2; string MdbGateway = 3; bool UseBearerForYdb = 4; - bool MdbTransformHost = 5 [deprecated = true]; // TODO: remove it in https://st.yandex-team.ru/YQ-2229 + bool MdbTransformHost = 5; string ObjectStorageEndpoint = 6; string IdsPrefix = 7; uint64 MaxTasksPerOperation = 8; diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt index 3c27829d986..20c89d07b0f 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt @@ -21,5 +21,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC ) target_sources(fq-libs-db_id_async_resolver_impl PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp ) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt index a26a3f5d590..5dadab7c257 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt @@ -22,5 +22,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC ) target_sources(fq-libs-db_id_async_resolver_impl PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp ) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt index a26a3f5d590..5dadab7c257 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt @@ -22,5 +22,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC ) target_sources(fq-libs-db_id_async_resolver_impl PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp ) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt index 3c27829d986..20c89d07b0f 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt @@ -21,5 +21,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC ) target_sources(fq-libs-db_id_async_resolver_impl PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp - ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp ) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp index 4621a2c4be0..97b9f1f40ab 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp @@ -10,13 +10,13 @@ TDatabaseAsyncResolverImpl::TDatabaseAsyncResolverImpl( const NActors::TActorId& recipient, const TString& ydbMvpEndpoint, const TString& mdbGateway, - NYql::IMdbHostTransformer::TPtr&& mdbHostTransformer, + NYql::IMdbEndpointGenerator::TPtr&& mdbEndpointGenerator, const TString& traceId) : ActorSystem(actorSystem) , Recipient(recipient) , YdbMvpEndpoint(ydbMvpEndpoint) , MdbGateway(mdbGateway) - , MdbHostTransformer(std::move(mdbHostTransformer)) + , mdbEndpointGenerator(std::move(mdbEndpointGenerator)) , TraceId(traceId) { } @@ -45,7 +45,7 @@ TFuture<NYql::TDatabaseResolverResponse> TDatabaseAsyncResolverImpl::ResolveIds( ActorSystem->Send(new NActors::IEventHandle(Recipient, callbackId, new TEvents::TEvEndpointRequest(ids, YdbMvpEndpoint, MdbGateway, - TraceId, MdbHostTransformer))); + TraceId, mdbEndpointGenerator))); return promise.GetFuture(); } diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h index 8feb0ecdcc8..9e86ab16c0b 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h @@ -2,7 +2,7 @@ #include <ydb/core/fq/libs/events/events.h> #include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> -#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h> +#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h> #include <ydb/library/yql/providers/dq/actors/actor_helpers.h> namespace NFq { @@ -14,7 +14,7 @@ public: const NActors::TActorId& recipient, const TString& ydbMvpEndpoint, const TString& mdbGateway, - NYql::IMdbHostTransformer::TPtr&& mdbHostTransformer, + NYql::IMdbEndpointGenerator::TPtr&& endpointGenerator, const TString& traceId = "" ); @@ -24,7 +24,7 @@ private: const NActors::TActorId Recipient; const TString YdbMvpEndpoint; const TString MdbGateway; - NYql::IMdbHostTransformer::TPtr MdbHostTransformer; + NYql::IMdbEndpointGenerator::TPtr mdbEndpointGenerator; const TString TraceId; }; diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp new file mode 100644 index 00000000000..152dc6c499c --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp @@ -0,0 +1,72 @@ +#include "mdb_endpoint_generator.h" + +namespace NFq { + + TString ReplaceDomain(const TString& mdbHost) { + return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net"; + } + + // TMdbEndpointGeneratorLegacy implements behavior required by YQL legacy ClickHouse provider + class TMdbEndpointGeneratorLegacy: public NYql::IMdbEndpointGenerator { + TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override { + // Inherited from here + // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r11819335#L27 + if (databaseType == NYql::EDatabaseType::ClickHouse) { + return ReplaceDomain(mdbHost) + ":8443"; + } + + ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); + } + }; + + NYql::IMdbEndpointGenerator::TPtr + MakeMdbEndpointGeneratorLegacy() { + return std::make_shared<TMdbEndpointGeneratorLegacy>(); + } + + // TMdbEndpointGeneratorGeneric implements behavior required by YQL Generic provider + // that interacts with data sources through a separate Connector service + class TMdbEndpointGeneratorGeneric: public NYql::IMdbEndpointGenerator { + public: + TMdbEndpointGeneratorGeneric(bool transformHost) + : TransformHost(transformHost) + { + } + + TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override { + auto fixedHost = TransformHost ? ReplaceDomain(mdbHost) : mdbHost; + + switch (databaseType) { + case NYql::EDatabaseType::ClickHouse: + // https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect + // TODO: fix Native protocol + TLS https://st.yandex-team.ru/YQ-2286 + return fixedHost + ":8443"; + case NYql::EDatabaseType::PostgreSQL: + // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect + return fixedHost + ":6432"; + default: + ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); + }; + } + + private: + bool TransformHost; + }; + + NYql::IMdbEndpointGenerator::TPtr + MakeMdbEndpointGeneratorGeneric(bool transformHost) { + return std::make_shared<TMdbEndpointGeneratorGeneric>(transformHost); + } + + // TMdbEndpointGeneratorNoop just does nothing + class TMdbEndpointGeneratorNoop: public NYql::IMdbEndpointGenerator { + TString ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost) const override { + return mdbHost; + } + }; + + NYql::IMdbEndpointGenerator::TPtr + MakeMdbEndpointGeneratorNoop() { + return std::make_shared<TMdbEndpointGeneratorNoop>(); + } +}
\ No newline at end of file diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h new file mode 100644 index 00000000000..f7be4f68de2 --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h @@ -0,0 +1,10 @@ +#pragma once + +#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> +#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h> + +namespace NFq { + NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorLegacy(); + NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorGeneric(bool transformHost); + NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorNoop(); +} diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp deleted file mode 100644 index 784405d050c..00000000000 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "mdb_host_transformer.h" - -namespace NFq { - // TMdbHostTransformerLegacy implements behavior required by YQL legacy ClickHouse provider - class TMdbHostTransformerLegacy: public NYql::IMdbHostTransformer { - TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override { - // Inherited from here - // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r11819335#L27 - if (databaseType == NYql::EDatabaseType::ClickHouse) { - return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net:8443"; - } - - ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); - } - }; - - NYql::IMdbHostTransformer::TPtr - MakeTMdbHostTransformerLegacy() { - return std::make_shared<TMdbHostTransformerLegacy>(); - } - - // TMdbHostTransformerGeneric implements behavior required by YQL Generic provider - // that interacts with data sources through a separate Connector service - class TMdbHostTransformerGeneric: public NYql::IMdbHostTransformer { - TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override { - switch (databaseType) { - case NYql::EDatabaseType::ClickHouse: - // https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect - // TODO: fix Native protocol + TLS https://st.yandex-team.ru/YQ-2286 - return mdbHost + ":8443"; - case NYql::EDatabaseType::PostgreSQL: - // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect - return mdbHost + ":6432"; - default: - ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); - }; - } - }; - - NYql::IMdbHostTransformer::TPtr - MakeTMdbHostTransformerGeneric() { - return std::make_shared<TMdbHostTransformerGeneric>(); - } - - // TMdbHostTransformerNoop just does nothing - class TMdbHostTransformerNoop: public NYql::IMdbHostTransformer { - TString ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost) const override { - return mdbHost; - } - }; - - NYql::IMdbHostTransformer::TPtr - MakeTMdbHostTransformerNoop() { - return std::make_shared<TMdbHostTransformerNoop>(); - } -}
\ No newline at end of file diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h deleted file mode 100644 index 4a26bc6b654..00000000000 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> -#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h> - -namespace NFq { - NYql::IMdbHostTransformer::TPtr MakeTMdbHostTransformerLegacy(); - NYql::IMdbHostTransformer::TPtr MakeTMdbHostTransformerGeneric(); - NYql::IMdbHostTransformer::TPtr MakeTMdbHostTransformerNoop(); -} diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make index 2ac916e03c6..49a3657e8fa 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make @@ -2,7 +2,7 @@ LIBRARY() SRCS( db_async_resolver_impl.cpp - mdb_host_transformer.cpp + mdb_endpoint_generator.cpp ) PEERDIR( diff --git a/ydb/core/fq/libs/events/events.h b/ydb/core/fq/libs/events/events.h index 62bc65299a6..9ac62b2bd43 100644 --- a/ydb/core/fq/libs/events/events.h +++ b/ydb/core/fq/libs/events/events.h @@ -3,7 +3,7 @@ #include <ydb/library/yql/core/facade/yql_facade.h> #include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> -#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h> +#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h> #include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h> #include <ydb/library/yql/public/issue/yql_issue.h> @@ -116,19 +116,19 @@ struct TEvents { TString YdbMvpEndpoint; TString MdbGateway; TString TraceId; - const NYql::IMdbHostTransformer::TPtr MdbHostTransformer; + const NYql::IMdbEndpointGenerator::TPtr MdbEndpointGenerator; TEvEndpointRequest( const NYql::IDatabaseAsyncResolver::TDatabaseAuthMap& databaseIds, const TString& ydbMvpEndpoint, const TString& mdbGateway, const TString& traceId, - const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer) + const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator) : DatabaseIds(databaseIds) , YdbMvpEndpoint(ydbMvpEndpoint) , MdbGateway(mdbGateway) , TraceId(traceId) - , MdbHostTransformer(mdbHostTransformer) + , MdbEndpointGenerator(mdbEndpointGenerator) { } }; diff --git a/ydb/core/fq/libs/test_connection/test_connection.cpp b/ydb/core/fq/libs/test_connection/test_connection.cpp index d2c2fb1ba9c..2d9621b4ada 100644 --- a/ydb/core/fq/libs/test_connection/test_connection.cpp +++ b/ydb/core/fq/libs/test_connection/test_connection.cpp @@ -9,7 +9,7 @@ #include <ydb/core/fq/libs/common/util.h> #include <ydb/core/fq/libs/config/yq_issue.h> #include <ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h> -#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h> +#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h> #include <ydb/core/fq/libs/control_plane_storage/config.h> #include <ydb/library/security/util.h> @@ -144,7 +144,7 @@ public: DbResolver = std::make_shared<NFq::TDatabaseAsyncResolverImpl>( NActors::TActivationContext::ActorSystem(), DatabaseResolverActor, CommonConfig.GetYdbMvpCloudEndpoint(), CommonConfig.GetMdbGateway(), - NFq::MakeTMdbHostTransformerGeneric() + NFq::MakeMdbEndpointGeneratorGeneric(CommonConfig.GetMdbTransformHost()) ); Become(&TTestConnectionActor::StateFunc); diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h index 188f739f984..a2132b45c16 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h +++ b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h @@ -3,15 +3,15 @@ #include "db_async_resolver.h" namespace NYql { - // IMdbHostTransformer is responsible for transforming the managed database instance host name + // IMdbEndpointGenerator is responsible for transforming the managed database instance hostname // into endpoint (`fqdn:port`) to establish network connection with data source. // The host names are obtained from MDB API, for example: // https://cloud.yandex.ru/docs/managed-clickhouse/api-ref/Cluster/listHosts - class IMdbHostTransformer { + class IMdbEndpointGenerator { public: - using TPtr = std::shared_ptr<IMdbHostTransformer>; + using TPtr = std::shared_ptr<IMdbEndpointGenerator>; virtual TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const = 0; - virtual ~IMdbHostTransformer() = default; + virtual ~IMdbEndpointGenerator() = default; }; } diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make index 9e226a741ad..f20098f9b28 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make +++ b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make @@ -2,7 +2,7 @@ LIBRARY() SRCS( db_async_resolver.h - mdb_host_transformer.h + mdb_endpoint_generator.h ) PEERDIR( diff --git a/ydb/services/fq/ut_integration/fq_ut.cpp b/ydb/services/fq/ut_integration/fq_ut.cpp index b80e4b6f9fd..cc343d4a42a 100644 --- a/ydb/services/fq/ut_integration/fq_ut.cpp +++ b/ydb/services/fq/ut_integration/fq_ut.cpp @@ -9,7 +9,7 @@ #include <ydb/core/fq/libs/control_plane_storage/message_builders.h> #include <ydb/core/fq/libs/actors/database_resolver.h> -#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h> +#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h> #include <ydb/library/yql/public/issue/yql_issue_message.h> @@ -860,7 +860,7 @@ Y_UNIT_TEST_SUITE(Yq_2) { // use fork for data test due to ch initialization problem Y_UNIT_TEST(Test_HostNameTrasformation) { { - auto transformer = ::NFq::MakeTMdbHostTransformerLegacy(); + auto transformer = ::NFq::MakeMdbEndpointGeneratorLegacy(); UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1c-p5waby2y5y1kb5ue.db.yandex.net"), "rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443"); UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "ya.ru"), @@ -868,12 +868,20 @@ Y_UNIT_TEST_SUITE(Yq_2) { } { - auto transformer = ::NFq::MakeTMdbHostTransformerGeneric(); - UNIT_ASSERT_VALUES_EQUAL(::NFq::MakeTMdbHostTransformerGeneric()->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"), + auto transformer = ::NFq::MakeMdbEndpointGeneratorGeneric(false); + UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"), "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net:8443"); - UNIT_ASSERT_VALUES_EQUAL(::NFq::MakeTMdbHostTransformerGeneric()->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net"), + UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net"), "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net:6432"); } + + { + auto transformer = ::NFq::MakeMdbEndpointGeneratorGeneric(true); + UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"), + "rc1a-d6dv17lv47v5mcop.db.yandex.net:8443"); + UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net"), + "rc1b-eyt6dtobu96rwydq.db.yandex.net:6432"); + } } SIMPLE_UNIT_FORKED_TEST(ReadFromYdbOverYq) { |