aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvitalyisaev <vitalyisaev@yandex-team.com>2023-08-24 21:21:48 +0300
committervitalyisaev <vitalyisaev@yandex-team.com>2023-08-24 21:59:13 +0300
commit24fce6d1ca9debced3e88aec3a404aa233a6c1c9 (patch)
treef6469161308affe21082411f8a4101cb32f72d45
parent7a0f7576d01bee0c781e115b219d34c475d74e67 (diff)
downloadydb-24fce6d1ca9debced3e88aec3a404aa233a6c1c9.tar.gz
YQ Connector: use MdbTransformHost config field again
В конфигурации YQ есть поле `MdbTransformHost`. Ранее оно было помечено как устаревшее, теперь снова используется.
-rw-r--r--ydb/core/fq/libs/actors/database_resolver.cpp26
-rw-r--r--ydb/core/fq/libs/actors/run_actor.cpp5
-rw-r--r--ydb/core/fq/libs/config/protos/common.proto2
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt2
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt2
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt2
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt2
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp6
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h6
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp72
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h10
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp56
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h10
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/ya.make2
-rw-r--r--ydb/core/fq/libs/events/events.h8
-rw-r--r--ydb/core/fq/libs/test_connection/test_connection.cpp4
-rw-r--r--ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h (renamed from ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h)8
-rw-r--r--ydb/library/yql/providers/common/db_id_async_resolver/ya.make2
-rw-r--r--ydb/services/fq/ut_integration/fq_ut.cpp18
19 files changed, 133 insertions, 110 deletions
diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp
index ee5aaccd8cd..9591ff9b51d 100644
--- a/ydb/core/fq/libs/actors/database_resolver.cpp
+++ b/ydb/core/fq/libs/actors/database_resolver.cpp
@@ -20,7 +20,7 @@ using namespace NYql;
using TEndpoint = NYql::TDatabaseResolverResponse::TEndpoint;
-using TParser = std::function<TEndpoint(NJson::TJsonValue& body, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer)>;
+using TParser = std::function<TEndpoint(NJson::TJsonValue& body, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator)>;
using TParsers = THashMap<NYql::EDatabaseType, TParser>;
using TCache = TTtlCache<std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>, std::variant<TEndpoint, TString>>;
@@ -38,13 +38,13 @@ public:
const TDatabaseResolverResponse::TDatabaseEndpointsMap& ready,
const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>>& requests,
const TString& traceId,
- const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer,
+ const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
const TParsers& parsers)
: Sender(sender)
, Cache(cache)
, Requests(requests)
, TraceId(traceId)
- , MdbHostTransformer(mdbHostTransformer)
+ , MdbEndpointGenerator(mdbEndpointGenerator)
, DatabaseId2Endpoint(ready)
, Parsers(parsers)
{ }
@@ -127,7 +127,7 @@ private:
TParsers::const_iterator parserIt;
if (parseJsonOk && (parserIt = Parsers.find(databaseType)) != Parsers.end()) {
try {
- auto res = parserIt->second(databaseInfo, MdbHostTransformer);
+ auto res = parserIt->second(databaseInfo, MdbEndpointGenerator);
LOG_D("Got db_id: " << databaseId
<< ", db type: " << static_cast<std::underlying_type<NYql::EDatabaseType>::type>(databaseType)
<< ", endpoint: " << res.Endpoint
@@ -184,7 +184,7 @@ private:
TCache& Cache;
const THashMap<NHttp::THttpOutgoingRequestPtr, std::tuple<TString, NYql::EDatabaseType, NYql::TDatabaseAuth>> Requests;
const TString TraceId;
- const NYql::IMdbHostTransformer::TPtr MdbHostTransformer;
+ const NYql::IMdbEndpointGenerator::TPtr MdbEndpointGenerator;
TDatabaseResolverResponse::TDatabaseEndpointsMap DatabaseId2Endpoint;
size_t HandledIds = 0;
bool Success = true;
@@ -205,7 +205,7 @@ public:
.SetErrorTtl(TDuration::Minutes(1))
.SetMaxSize(1000000))
{
- auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr&) {
+ auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&) {
bool secure = false;
TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust();
TString prefix("/?database=");
@@ -227,9 +227,9 @@ public:
return TEndpoint{endpoint, database, secure};
};
Parsers[NYql::EDatabaseType::Ydb] = ydbParser;
- Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer)
+ Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator)
{
- auto ret = ydbParser(databaseInfo, mdbHostTransformer);
+ auto ret = ydbParser(databaseInfo, mdbEndpointGenerator);
// TODO: Take explicit field from MVP
if (ret.Endpoint.StartsWith("ydb.")) {
// Replace "ydb." -> "yds."
@@ -237,7 +237,7 @@ public:
}
return ret;
};
- Parsers[NYql::EDatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer) {
+ Parsers[NYql::EDatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator) {
TString endpoint;
TVector<TString> aliveHosts;
for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) {
@@ -246,7 +246,7 @@ public:
}
}
if (!aliveHosts.empty()) {
- endpoint = mdbHostTransformer->ToEndpoint(
+ endpoint = mdbEndpointGenerator->ToEndpoint(
NYql::EDatabaseType::ClickHouse,
aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())]
);
@@ -257,7 +257,7 @@ public:
return TEndpoint{endpoint, "", true};
};
- Parsers[NYql::EDatabaseType::PostgreSQL] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer) {
+ Parsers[NYql::EDatabaseType::PostgreSQL] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator) {
TString endpoint;
TVector<TString> aliveHosts;
for (const auto& host : databaseInfo.GetMap().at("hosts").GetArraySafe()) {
@@ -276,7 +276,7 @@ public:
}
}
if (!aliveHosts.empty()) {
- endpoint = mdbHostTransformer->ToEndpoint(
+ endpoint = mdbEndpointGenerator->ToEndpoint(
NYql::EDatabaseType::PostgreSQL,
aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())]
);
@@ -375,7 +375,7 @@ private:
if (!requests.empty()) {
auto helper = Register(
- new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbHostTransformer, Parsers));
+ new TResponseProcessor(ev->Sender, Cache, ready, requests, TraceId, ev->Get()->MdbEndpointGenerator, Parsers));
for (const auto& [request, _] : requests) {
TActivationContext::Send(new IEventHandle(HttpProxy, helper, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest(request)));
diff --git a/ydb/core/fq/libs/actors/run_actor.cpp b/ydb/core/fq/libs/actors/run_actor.cpp
index f2977c626ef..7fea6502a66 100644
--- a/ydb/core/fq/libs/actors/run_actor.cpp
+++ b/ydb/core/fq/libs/actors/run_actor.cpp
@@ -59,7 +59,7 @@
#include <ydb/core/fq/libs/control_plane_storage/events/events.h>
#include <ydb/core/fq/libs/control_plane_storage/util.h>
#include <ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h>
-#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h>
+#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h>
#include <ydb/core/fq/libs/gateway/empty_gateway.h>
#include <ydb/core/fq/libs/private_client/events.h>
#include <ydb/core/fq/libs/private_client/private_client.h>
@@ -1832,8 +1832,7 @@ private:
Params.DatabaseResolver,
Params.Config.GetCommon().GetYdbMvpCloudEndpoint(),
Params.Config.GetCommon().GetMdbGateway(),
- NFq::MakeTMdbHostTransformerGeneric(),
- // Params.Config.GetCommon().GetMdbTransformHost(),
+ NFq::MakeMdbEndpointGeneratorGeneric(Params.Config.GetCommon().GetMdbTransformHost()),
Params.QueryId);
{
// TBD: move init to better place
diff --git a/ydb/core/fq/libs/config/protos/common.proto b/ydb/core/fq/libs/config/protos/common.proto
index ade8d5562c7..7bb6da3577b 100644
--- a/ydb/core/fq/libs/config/protos/common.proto
+++ b/ydb/core/fq/libs/config/protos/common.proto
@@ -17,7 +17,7 @@ message TCommonConfig {
string YdbMvpCloudEndpoint = 2;
string MdbGateway = 3;
bool UseBearerForYdb = 4;
- bool MdbTransformHost = 5 [deprecated = true]; // TODO: remove it in https://st.yandex-team.ru/YQ-2229
+ bool MdbTransformHost = 5;
string ObjectStorageEndpoint = 6;
string IdsPrefix = 7;
uint64 MaxTasksPerOperation = 8;
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt
index 3c27829d986..20c89d07b0f 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.darwin-x86_64.txt
@@ -21,5 +21,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC
)
target_sources(fq-libs-db_id_async_resolver_impl PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
)
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt
index a26a3f5d590..5dadab7c257 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-aarch64.txt
@@ -22,5 +22,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC
)
target_sources(fq-libs-db_id_async_resolver_impl PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
)
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt
index a26a3f5d590..5dadab7c257 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.linux-x86_64.txt
@@ -22,5 +22,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC
)
target_sources(fq-libs-db_id_async_resolver_impl PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
)
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt
index 3c27829d986..20c89d07b0f 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/CMakeLists.windows-x86_64.txt
@@ -21,5 +21,5 @@ target_link_libraries(fq-libs-db_id_async_resolver_impl PUBLIC
)
target_sources(fq-libs-db_id_async_resolver_impl PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp
- ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
)
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp
index 4621a2c4be0..97b9f1f40ab 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.cpp
@@ -10,13 +10,13 @@ TDatabaseAsyncResolverImpl::TDatabaseAsyncResolverImpl(
const NActors::TActorId& recipient,
const TString& ydbMvpEndpoint,
const TString& mdbGateway,
- NYql::IMdbHostTransformer::TPtr&& mdbHostTransformer,
+ NYql::IMdbEndpointGenerator::TPtr&& mdbEndpointGenerator,
const TString& traceId)
: ActorSystem(actorSystem)
, Recipient(recipient)
, YdbMvpEndpoint(ydbMvpEndpoint)
, MdbGateway(mdbGateway)
- , MdbHostTransformer(std::move(mdbHostTransformer))
+ , mdbEndpointGenerator(std::move(mdbEndpointGenerator))
, TraceId(traceId)
{
}
@@ -45,7 +45,7 @@ TFuture<NYql::TDatabaseResolverResponse> TDatabaseAsyncResolverImpl::ResolveIds(
ActorSystem->Send(new NActors::IEventHandle(Recipient, callbackId,
new TEvents::TEvEndpointRequest(ids, YdbMvpEndpoint, MdbGateway,
- TraceId, MdbHostTransformer)));
+ TraceId, mdbEndpointGenerator)));
return promise.GetFuture();
}
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h
index 8feb0ecdcc8..9e86ab16c0b 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h
@@ -2,7 +2,7 @@
#include <ydb/core/fq/libs/events/events.h>
#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h>
-#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h>
+#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h>
#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
namespace NFq {
@@ -14,7 +14,7 @@ public:
const NActors::TActorId& recipient,
const TString& ydbMvpEndpoint,
const TString& mdbGateway,
- NYql::IMdbHostTransformer::TPtr&& mdbHostTransformer,
+ NYql::IMdbEndpointGenerator::TPtr&& endpointGenerator,
const TString& traceId = ""
);
@@ -24,7 +24,7 @@ private:
const NActors::TActorId Recipient;
const TString YdbMvpEndpoint;
const TString MdbGateway;
- NYql::IMdbHostTransformer::TPtr MdbHostTransformer;
+ NYql::IMdbEndpointGenerator::TPtr mdbEndpointGenerator;
const TString TraceId;
};
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
new file mode 100644
index 00000000000..152dc6c499c
--- /dev/null
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
@@ -0,0 +1,72 @@
+#include "mdb_endpoint_generator.h"
+
+namespace NFq {
+
+ TString ReplaceDomain(const TString& mdbHost) {
+ return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net";
+ }
+
+ // TMdbEndpointGeneratorLegacy implements behavior required by YQL legacy ClickHouse provider
+ class TMdbEndpointGeneratorLegacy: public NYql::IMdbEndpointGenerator {
+ TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override {
+ // Inherited from here
+ // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r11819335#L27
+ if (databaseType == NYql::EDatabaseType::ClickHouse) {
+ return ReplaceDomain(mdbHost) + ":8443";
+ }
+
+ ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType);
+ }
+ };
+
+ NYql::IMdbEndpointGenerator::TPtr
+ MakeMdbEndpointGeneratorLegacy() {
+ return std::make_shared<TMdbEndpointGeneratorLegacy>();
+ }
+
+ // TMdbEndpointGeneratorGeneric implements behavior required by YQL Generic provider
+ // that interacts with data sources through a separate Connector service
+ class TMdbEndpointGeneratorGeneric: public NYql::IMdbEndpointGenerator {
+ public:
+ TMdbEndpointGeneratorGeneric(bool transformHost)
+ : TransformHost(transformHost)
+ {
+ }
+
+ TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override {
+ auto fixedHost = TransformHost ? ReplaceDomain(mdbHost) : mdbHost;
+
+ switch (databaseType) {
+ case NYql::EDatabaseType::ClickHouse:
+ // https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect
+ // TODO: fix Native protocol + TLS https://st.yandex-team.ru/YQ-2286
+ return fixedHost + ":8443";
+ case NYql::EDatabaseType::PostgreSQL:
+ // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect
+ return fixedHost + ":6432";
+ default:
+ ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType);
+ };
+ }
+
+ private:
+ bool TransformHost;
+ };
+
+ NYql::IMdbEndpointGenerator::TPtr
+ MakeMdbEndpointGeneratorGeneric(bool transformHost) {
+ return std::make_shared<TMdbEndpointGeneratorGeneric>(transformHost);
+ }
+
+ // TMdbEndpointGeneratorNoop just does nothing
+ class TMdbEndpointGeneratorNoop: public NYql::IMdbEndpointGenerator {
+ TString ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost) const override {
+ return mdbHost;
+ }
+ };
+
+ NYql::IMdbEndpointGenerator::TPtr
+ MakeMdbEndpointGeneratorNoop() {
+ return std::make_shared<TMdbEndpointGeneratorNoop>();
+ }
+} \ No newline at end of file
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h
new file mode 100644
index 00000000000..f7be4f68de2
--- /dev/null
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h>
+#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h>
+
+namespace NFq {
+ NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorLegacy();
+ NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorGeneric(bool transformHost);
+ NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorNoop();
+}
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
deleted file mode 100644
index 784405d050c..00000000000
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-#include "mdb_host_transformer.h"
-
-namespace NFq {
- // TMdbHostTransformerLegacy implements behavior required by YQL legacy ClickHouse provider
- class TMdbHostTransformerLegacy: public NYql::IMdbHostTransformer {
- TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override {
- // Inherited from here
- // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r11819335#L27
- if (databaseType == NYql::EDatabaseType::ClickHouse) {
- return mdbHost.substr(0, mdbHost.find('.')) + ".db.yandex.net:8443";
- }
-
- ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType);
- }
- };
-
- NYql::IMdbHostTransformer::TPtr
- MakeTMdbHostTransformerLegacy() {
- return std::make_shared<TMdbHostTransformerLegacy>();
- }
-
- // TMdbHostTransformerGeneric implements behavior required by YQL Generic provider
- // that interacts with data sources through a separate Connector service
- class TMdbHostTransformerGeneric: public NYql::IMdbHostTransformer {
- TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override {
- switch (databaseType) {
- case NYql::EDatabaseType::ClickHouse:
- // https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect
- // TODO: fix Native protocol + TLS https://st.yandex-team.ru/YQ-2286
- return mdbHost + ":8443";
- case NYql::EDatabaseType::PostgreSQL:
- // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect
- return mdbHost + ":6432";
- default:
- ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType);
- };
- }
- };
-
- NYql::IMdbHostTransformer::TPtr
- MakeTMdbHostTransformerGeneric() {
- return std::make_shared<TMdbHostTransformerGeneric>();
- }
-
- // TMdbHostTransformerNoop just does nothing
- class TMdbHostTransformerNoop: public NYql::IMdbHostTransformer {
- TString ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost) const override {
- return mdbHost;
- }
- };
-
- NYql::IMdbHostTransformer::TPtr
- MakeTMdbHostTransformerNoop() {
- return std::make_shared<TMdbHostTransformerNoop>();
- }
-} \ No newline at end of file
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h
deleted file mode 100644
index 4a26bc6b654..00000000000
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#pragma once
-
-#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h>
-#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h>
-
-namespace NFq {
- NYql::IMdbHostTransformer::TPtr MakeTMdbHostTransformerLegacy();
- NYql::IMdbHostTransformer::TPtr MakeTMdbHostTransformerGeneric();
- NYql::IMdbHostTransformer::TPtr MakeTMdbHostTransformerNoop();
-}
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make
index 2ac916e03c6..49a3657e8fa 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make
@@ -2,7 +2,7 @@ LIBRARY()
SRCS(
db_async_resolver_impl.cpp
- mdb_host_transformer.cpp
+ mdb_endpoint_generator.cpp
)
PEERDIR(
diff --git a/ydb/core/fq/libs/events/events.h b/ydb/core/fq/libs/events/events.h
index 62bc65299a6..9ac62b2bd43 100644
--- a/ydb/core/fq/libs/events/events.h
+++ b/ydb/core/fq/libs/events/events.h
@@ -3,7 +3,7 @@
#include <ydb/library/yql/core/facade/yql_facade.h>
#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h>
-#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h>
+#include <ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h>
#include <ydb/library/yql/providers/dq/provider/yql_dq_gateway.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
@@ -116,19 +116,19 @@ struct TEvents {
TString YdbMvpEndpoint;
TString MdbGateway;
TString TraceId;
- const NYql::IMdbHostTransformer::TPtr MdbHostTransformer;
+ const NYql::IMdbEndpointGenerator::TPtr MdbEndpointGenerator;
TEvEndpointRequest(
const NYql::IDatabaseAsyncResolver::TDatabaseAuthMap& databaseIds,
const TString& ydbMvpEndpoint,
const TString& mdbGateway,
const TString& traceId,
- const NYql::IMdbHostTransformer::TPtr& mdbHostTransformer)
+ const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator)
: DatabaseIds(databaseIds)
, YdbMvpEndpoint(ydbMvpEndpoint)
, MdbGateway(mdbGateway)
, TraceId(traceId)
- , MdbHostTransformer(mdbHostTransformer)
+ , MdbEndpointGenerator(mdbEndpointGenerator)
{ }
};
diff --git a/ydb/core/fq/libs/test_connection/test_connection.cpp b/ydb/core/fq/libs/test_connection/test_connection.cpp
index d2c2fb1ba9c..2d9621b4ada 100644
--- a/ydb/core/fq/libs/test_connection/test_connection.cpp
+++ b/ydb/core/fq/libs/test_connection/test_connection.cpp
@@ -9,7 +9,7 @@
#include <ydb/core/fq/libs/common/util.h>
#include <ydb/core/fq/libs/config/yq_issue.h>
#include <ydb/core/fq/libs/db_id_async_resolver_impl/db_async_resolver_impl.h>
-#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h>
+#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h>
#include <ydb/core/fq/libs/control_plane_storage/config.h>
#include <ydb/library/security/util.h>
@@ -144,7 +144,7 @@ public:
DbResolver = std::make_shared<NFq::TDatabaseAsyncResolverImpl>(
NActors::TActivationContext::ActorSystem(), DatabaseResolverActor,
CommonConfig.GetYdbMvpCloudEndpoint(), CommonConfig.GetMdbGateway(),
- NFq::MakeTMdbHostTransformerGeneric()
+ NFq::MakeMdbEndpointGeneratorGeneric(CommonConfig.GetMdbTransformHost())
);
Become(&TTestConnectionActor::StateFunc);
diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h
index 188f739f984..a2132b45c16 100644
--- a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_host_transformer.h
+++ b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h
@@ -3,15 +3,15 @@
#include "db_async_resolver.h"
namespace NYql {
- // IMdbHostTransformer is responsible for transforming the managed database instance host name
+ // IMdbEndpointGenerator is responsible for transforming the managed database instance hostname
// into endpoint (`fqdn:port`) to establish network connection with data source.
// The host names are obtained from MDB API, for example:
// https://cloud.yandex.ru/docs/managed-clickhouse/api-ref/Cluster/listHosts
- class IMdbHostTransformer {
+ class IMdbEndpointGenerator {
public:
- using TPtr = std::shared_ptr<IMdbHostTransformer>;
+ using TPtr = std::shared_ptr<IMdbEndpointGenerator>;
virtual TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const = 0;
- virtual ~IMdbHostTransformer() = default;
+ virtual ~IMdbEndpointGenerator() = default;
};
}
diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make
index 9e226a741ad..f20098f9b28 100644
--- a/ydb/library/yql/providers/common/db_id_async_resolver/ya.make
+++ b/ydb/library/yql/providers/common/db_id_async_resolver/ya.make
@@ -2,7 +2,7 @@ LIBRARY()
SRCS(
db_async_resolver.h
- mdb_host_transformer.h
+ mdb_endpoint_generator.h
)
PEERDIR(
diff --git a/ydb/services/fq/ut_integration/fq_ut.cpp b/ydb/services/fq/ut_integration/fq_ut.cpp
index b80e4b6f9fd..cc343d4a42a 100644
--- a/ydb/services/fq/ut_integration/fq_ut.cpp
+++ b/ydb/services/fq/ut_integration/fq_ut.cpp
@@ -9,7 +9,7 @@
#include <ydb/core/fq/libs/control_plane_storage/message_builders.h>
#include <ydb/core/fq/libs/actors/database_resolver.h>
-#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.h>
+#include <ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h>
#include <ydb/library/yql/public/issue/yql_issue_message.h>
@@ -860,7 +860,7 @@ Y_UNIT_TEST_SUITE(Yq_2) {
// use fork for data test due to ch initialization problem
Y_UNIT_TEST(Test_HostNameTrasformation) {
{
- auto transformer = ::NFq::MakeTMdbHostTransformerLegacy();
+ auto transformer = ::NFq::MakeMdbEndpointGeneratorLegacy();
UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1c-p5waby2y5y1kb5ue.db.yandex.net"),
"rc1c-p5waby2y5y1kb5ue.db.yandex.net:8443");
UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "ya.ru"),
@@ -868,12 +868,20 @@ Y_UNIT_TEST_SUITE(Yq_2) {
}
{
- auto transformer = ::NFq::MakeTMdbHostTransformerGeneric();
- UNIT_ASSERT_VALUES_EQUAL(::NFq::MakeTMdbHostTransformerGeneric()->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"),
+ auto transformer = ::NFq::MakeMdbEndpointGeneratorGeneric(false);
+ UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"),
"rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net:8443");
- UNIT_ASSERT_VALUES_EQUAL(::NFq::MakeTMdbHostTransformerGeneric()->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net"),
+ UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net"),
"rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net:6432");
}
+
+ {
+ auto transformer = ::NFq::MakeMdbEndpointGeneratorGeneric(true);
+ UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"),
+ "rc1a-d6dv17lv47v5mcop.db.yandex.net:8443");
+ UNIT_ASSERT_VALUES_EQUAL(transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net"),
+ "rc1b-eyt6dtobu96rwydq.db.yandex.net:6432");
+ }
}
SIMPLE_UNIT_FORKED_TEST(ReadFromYdbOverYq) {