aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvitalyisaev <vitalyisaev@ydb.tech>2023-11-24 10:32:28 +0300
committervitalyisaev <vitalyisaev@ydb.tech>2023-11-24 10:52:28 +0300
commitd3e4beea087cbfdd703ef23136c307dd67ca9ae0 (patch)
tree7e897a1599d151f7b1d63e5e3b37eef95e8b4384
parentb04e3c5ba75a4d9130991f7e3ed55b037d9275cb (diff)
downloadydb-d3e4beea087cbfdd703ef23136c307dd67ca9ae0.tar.gz
YQ Connector: use state of connection instead of service config when determining protocol
-rw-r--r--ydb/core/fq/libs/actors/database_resolver.cpp63
-rw-r--r--ydb/core/fq/libs/compute/common/run_actor_params.cpp2
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp54
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h2
-rw-r--r--ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp95
-rw-r--r--ydb/core/fq/libs/test_connection/test_connection.cpp2
-rw-r--r--ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp11
-rw-r--r--ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h15
-rw-r--r--ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h13
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/logger.go1
-rw-r--r--ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp32
-rw-r--r--ydb/library/yql/providers/generic/provider/yql_generic_settings.h1
12 files changed, 208 insertions, 83 deletions
diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp
index 8e3a50ea46..3ec5343da5 100644
--- a/ydb/core/fq/libs/actors/database_resolver.cpp
+++ b/ydb/core/fq/libs/actors/database_resolver.cpp
@@ -22,7 +22,12 @@ using namespace NActors;
using namespace NYql;
using TDatabaseDescription = NYql::TDatabaseResolverResponse::TDatabaseDescription;
-using TParser = std::function<TDatabaseDescription(NJson::TJsonValue& body, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls)>;
+using TParser = std::function<TDatabaseDescription(
+ NJson::TJsonValue& body,
+ const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
+ bool useTls,
+ NConnector::NApi::EProtocol protocol
+)>;
using TParsers = THashMap<NYql::EDatabaseType, TParser>;
struct TResolveParams {
@@ -144,7 +149,11 @@ private:
TParsers::const_iterator parserIt;
if (parseJsonOk && (parserIt = Parsers.find(params.DatabaseType)) != Parsers.end()) {
try {
- auto description = parserIt->second(databaseInfo, MdbEndpointGenerator, params.DatabaseAuth.UseTls);
+ auto description = parserIt->second(
+ databaseInfo,
+ MdbEndpointGenerator,
+ params.DatabaseAuth.UseTls,
+ params.DatabaseAuth.Protocol);
LOG_D("ResponseProcessor::Handle(HttpIncomingResponse): got description" << ": params: " << params.ToDebugString()
<< ", description: " << description.ToDebugString());
DatabaseId2Description[std::make_pair(params.Id, params.DatabaseType)] = description;
@@ -223,7 +232,7 @@ public:
.SetErrorTtl(TDuration::Minutes(1))
.SetMaxSize(1000000))
{
- auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool) {
+ auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NConnector::NApi::EProtocol) {
bool secure = false;
TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust();
TString prefix("/?database=");
@@ -245,10 +254,14 @@ public:
return TDatabaseDescription{endpoint, "", 0, database, secure};
};
Parsers[NYql::EDatabaseType::Ydb] = ydbParser;
- Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls)
+ Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](
+ NJson::TJsonValue& databaseInfo,
+ const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
+ bool useTls,
+ NConnector::NApi::EProtocol protocol)
{
bool isDedicatedDb = databaseInfo.GetMap().contains("storageConfig");
- auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls);
+ auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol);
// TODO: Take explicit field from MVP
if (!isDedicatedDb && ret.Endpoint.StartsWith("ydb.")) {
// Replace "ydb." -> "yds."
@@ -256,7 +269,12 @@ public:
}
return ret;
};
- Parsers[NYql::EDatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) {
+ Parsers[NYql::EDatabaseType::ClickHouse] = [](
+ NJson::TJsonValue& databaseInfo,
+ const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
+ bool useTls,
+ NConnector::NApi::EProtocol protocol
+ ) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
@@ -270,16 +288,24 @@ public:
ythrow yexception() << "No ALIVE ClickHouse hosts found";
}
- endpoint = mdbEndpointGenerator->ToEndpoint(
- NYql::EDatabaseType::ClickHouse,
- aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())],
- useTls
- );
+ NYql::IMdbEndpointGenerator::TParams params = {
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())],
+ .UseTls = useTls,
+ .Protocol = protocol,
+ };
+
+ endpoint = mdbEndpointGenerator->ToEndpoint(params);
return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls};
};
- Parsers[NYql::EDatabaseType::PostgreSQL] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) {
+ Parsers[NYql::EDatabaseType::PostgreSQL] = [](
+ NJson::TJsonValue& databaseInfo,
+ const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator,
+ bool useTls,
+ NConnector::NApi::EProtocol protocol
+ ) {
NYql::IMdbEndpointGenerator::TEndpoint endpoint;
TVector<TString> aliveHosts;
@@ -302,11 +328,14 @@ public:
ythrow yexception() << "No ALIVE PostgreSQL hosts found";
}
- endpoint = mdbEndpointGenerator->ToEndpoint(
- NYql::EDatabaseType::PostgreSQL,
- aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())],
- useTls
- );
+ NYql::IMdbEndpointGenerator::TParams params = {
+ .DatabaseType = NYql::EDatabaseType::PostgreSQL,
+ .MdbHost = aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())],
+ .UseTls = useTls,
+ .Protocol = protocol,
+ };
+
+ endpoint = mdbEndpointGenerator->ToEndpoint(params);
return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls};
};
diff --git a/ydb/core/fq/libs/compute/common/run_actor_params.cpp b/ydb/core/fq/libs/compute/common/run_actor_params.cpp
index 048540c2ed..1fa989e1ac 100644
--- a/ydb/core/fq/libs/compute/common/run_actor_params.cpp
+++ b/ydb/core/fq/libs/compute/common/run_actor_params.cpp
@@ -73,7 +73,7 @@ TRunActorParams::TRunActorParams(
, Scope(scope)
, AuthToken(authToken)
, DatabaseResolver(databaseResolver)
- , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(config.GetCommon().GetMdbTransformHost(), config.GetCommon().GetUseNativeProtocolForClickHouse()))
+ , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(config.GetCommon().GetMdbTransformHost()))
, QueryId(queryId)
, UserId(userId)
, Owner(owner)
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
index 88466ce081..b957a5052e 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp
@@ -18,15 +18,15 @@ namespace NFq {
// TMdbEndpointGeneratorLegacy implements behavior required by YQL legacy ClickHouse provider
class TMdbEndpointGeneratorLegacy: public NYql::IMdbEndpointGenerator {
- TEndpoint ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override {
+ TEndpoint ToEndpoint(const NYql::IMdbEndpointGenerator::TParams& params) const override {
// Inherited from here
// https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r11819335#L27
- if (databaseType == NYql::EDatabaseType::ClickHouse) {
- auto port = useTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT;
- return TEndpoint(ReplaceDomain(mdbHost), port);
+ if (params.DatabaseType == NYql::EDatabaseType::ClickHouse) {
+ auto port = params.UseTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT;
+ return TEndpoint(ReplaceDomain(params.MdbHost), port);
}
- ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType);
+ ythrow yexception() << "Unexpected database type: " << ToString(params.DatabaseType);
}
};
@@ -39,48 +39,58 @@ namespace NFq {
// that interacts with data sources through a separate Connector service
class TMdbEndpointGeneratorGeneric: public NYql::IMdbEndpointGenerator {
public:
- TMdbEndpointGeneratorGeneric(bool transformHost, bool useNativeProtocolForClickHouse = false)
- : TransformHost(transformHost), UseNativeProtocolForClickHouse(useNativeProtocolForClickHouse)
+ TMdbEndpointGeneratorGeneric(bool transformHost)
+ : TransformHost(transformHost)
{
}
- TEndpoint ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override {
- auto fixedHost = TransformHost ? ReplaceDomain(mdbHost) : mdbHost;
+ TEndpoint ToEndpoint(const NYql::IMdbEndpointGenerator::TParams& params) const override {
+ auto fixedHost = TransformHost ? ReplaceDomain(params.MdbHost) : params.MdbHost;
- switch (databaseType) {
+ switch (params.DatabaseType) {
case NYql::EDatabaseType::ClickHouse: {
// https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect
ui32 port;
- if (UseNativeProtocolForClickHouse) {
- port = useTls ? CLICKHOUSE_NATIVE_SECURE_PORT : CLICKHOUSE_NATIVE_INSECURE_PORT;
- } else {
- port = useTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT;
+
+ switch (params.Protocol) {
+ case NYql::NConnector::NApi::EProtocol::NATIVE:
+ port = params.UseTls ? CLICKHOUSE_NATIVE_SECURE_PORT : CLICKHOUSE_NATIVE_INSECURE_PORT;
+ break;
+ case NYql::NConnector::NApi::EProtocol::HTTP:
+ port = params.UseTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT;
+ break;
+ default:
+ ythrow yexception() << "Unexpected protocol for ClickHouse: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol);
}
return TEndpoint(fixedHost, port);
}
case NYql::EDatabaseType::PostgreSQL:
// https://cloud.yandex.ru/docs/managed-postgresql/operations/connect
- return TEndpoint(fixedHost, POSTGRESQL_PORT);
+ switch (params.Protocol) {
+ case NYql::NConnector::NApi::EProtocol::NATIVE:
+ return TEndpoint(fixedHost, POSTGRESQL_PORT);
+ default:
+ ythrow yexception() << "Unexpected protocol for PostgreSQL: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol);
+ }
default:
- ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType);
+ ythrow yexception() << "Unexpected database type: " << ToString(params.DatabaseType);
};
}
private:
bool TransformHost;
- bool UseNativeProtocolForClickHouse;
};
NYql::IMdbEndpointGenerator::TPtr
- MakeMdbEndpointGeneratorGeneric(bool transformHost, bool useNativeProtocolForClickHouse) {
- return std::make_shared<TMdbEndpointGeneratorGeneric>(transformHost, useNativeProtocolForClickHouse);
+ MakeMdbEndpointGeneratorGeneric(bool transformHost) {
+ return std::make_shared<TMdbEndpointGeneratorGeneric>(transformHost);
}
// TMdbEndpointGeneratorNoop just does nothing
class TMdbEndpointGeneratorNoop: public NYql::IMdbEndpointGenerator {
- TEndpoint ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost, bool) const override {
- return TEndpoint(mdbHost, 0);
+ TEndpoint ToEndpoint(const NYql::IMdbEndpointGenerator::TParams& params) const override {
+ return TEndpoint(params.MdbHost, 0);
}
};
@@ -88,4 +98,4 @@ namespace NFq {
MakeMdbEndpointGeneratorNoop() {
return std::make_shared<TMdbEndpointGeneratorNoop>();
}
-} \ No newline at end of file
+}
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h
index 75332332cb..f7be4f68de 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h
@@ -5,6 +5,6 @@
namespace NFq {
NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorLegacy();
- NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorGeneric(bool transformHost, bool useNativeProtocolForClickHouse = false);
+ NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorGeneric(bool transformHost);
NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorNoop();
}
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp
index 687570c5de..f02d3a4e08 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp
@@ -7,48 +7,115 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) {
Y_UNIT_TEST(Legacy) {
auto transformer = NFq::MakeMdbEndpointGeneratorLegacy();
+ auto params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = "rc1c-p5waby2y5y1kb5ue.db.yandex.net",
+ .UseTls = true,
+ .Protocol = NYql::NConnector::NApi::EProtocol::HTTP,
+ };
+
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1c-p5waby2y5y1kb5ue.db.yandex.net", true),
+ transformer->ToEndpoint(params),
TEndpoint("rc1c-p5waby2y5y1kb5ue.db.yandex.net", 8443));
+ params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = "ya.ru",
+ .UseTls = false,
+ .Protocol = NYql::NConnector::NApi::EProtocol::HTTP,
+ };
+
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "ya.ru", false),
+ transformer->ToEndpoint(params),
TEndpoint("ya.db.yandex.net", 8123));
}
Y_UNIT_TEST(Generic_NoTransformHost) {
auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(false);
+ auto params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net",
+ .UseTls = true,
+ .Protocol = NYql::NConnector::NApi::EProtocol::HTTP,
+ };
+
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", true),
+ transformer->ToEndpoint(params),
TEndpoint("rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", 8443));
+ params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::PostgreSQL,
+ .MdbHost = "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net",
+ .UseTls = false,
+ .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE,
+ };
+
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", false),
+ transformer->ToEndpoint(params),
TEndpoint("rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", 6432));
}
Y_UNIT_TEST(Generic_WithTransformHost) {
auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(true);
+ // ClickHouse
+
+ auto params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net",
+ .UseTls = false,
+ .Protocol = NYql::NConnector::NApi::EProtocol::HTTP,
+ };
+
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", false),
+ transformer->ToEndpoint(params),
TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 8123));
+ params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net",
+ .UseTls = false,
+ .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE,
+ };
+
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", true),
- TEndpoint("rc1b-eyt6dtobu96rwydq.db.yandex.net", 6432));
- }
+ transformer->ToEndpoint(params),
+ TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 9000));
- Y_UNIT_TEST(Generic_ClickhouseNative) {
- auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(false, true);
+ params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net",
+ .UseTls = true,
+ .Protocol = NYql::NConnector::NApi::EProtocol::HTTP,
+ };
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", true),
- TEndpoint("rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", 9440));
+ transformer->ToEndpoint(params),
+ TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 8443));
+
+ params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::ClickHouse,
+ .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net",
+ .UseTls = true,
+ .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE,
+ };
UNIT_ASSERT_VALUES_EQUAL(
- transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", false),
- TEndpoint("rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", 9000));
+ transformer->ToEndpoint(params),
+ TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 9440));
+
+ // PostgreSQL
+
+ params = NYql::IMdbEndpointGenerator::TParams{
+ .DatabaseType = NYql::EDatabaseType::PostgreSQL,
+ .MdbHost = "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net",
+ .UseTls = true,
+ .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE,
+ };
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ transformer->ToEndpoint(params),
+ TEndpoint("rc1b-eyt6dtobu96rwydq.db.yandex.net", 6432));
}
}
diff --git a/ydb/core/fq/libs/test_connection/test_connection.cpp b/ydb/core/fq/libs/test_connection/test_connection.cpp
index fd67251891..521442ade4 100644
--- a/ydb/core/fq/libs/test_connection/test_connection.cpp
+++ b/ydb/core/fq/libs/test_connection/test_connection.cpp
@@ -132,7 +132,7 @@ public:
, Counters(counters)
, Signer(signer)
, HttpGateway(httpGateway)
- , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(commonConfig.GetMdbTransformHost(), commonConfig.GetUseNativeProtocolForClickHouse()))
+ , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(commonConfig.GetMdbTransformHost()))
{}
static constexpr char ActorName[] = "YQ_TEST_CONNECTION";
diff --git a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp
index 345ea4b5ea..ea5004f832 100644
--- a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp
+++ b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp
@@ -75,16 +75,7 @@ namespace NKikimr::NKqp {
ConnectorClient = NYql::NConnector::MakeClientGRPC(GenericGatewaysConfig.GetConnector());
if (queryServiceConfig.HasMdbTransformHost()) {
- auto mdbTransformHost = queryServiceConfig.GetMdbTransformHost();
- bool useNativeProtocolForClickHouse = false;
- for (const auto& p: queryServiceConfig.GetGeneric().GetDefaultSettings()) {
- if (p.GetName() == "UseNativeProtocolForClickHouse") {
- TryFromString<bool>(p.GetValue(), useNativeProtocolForClickHouse);
- break;
- }
- }
-
- MdbEndpointGenerator = NFq::MakeMdbEndpointGeneratorGeneric(mdbTransformHost, useNativeProtocolForClickHouse);
+ MdbEndpointGenerator = NFq::MakeMdbEndpointGeneratorGeneric(queryServiceConfig.GetMdbTransformHost());
}
// Create actors required for MDB database resolving
diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h
index 0e2fc958c6..82bdb4f0ca 100644
--- a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h
+++ b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h
@@ -53,14 +53,19 @@ inline TString DatabaseTypeToMdbUrlPath(EDatabaseType databaseType) {
struct TDatabaseAuth {
// Serialized token value used to access MDB API
- TString StructuredToken;
- bool AddBearerToToken = false;
+ TString StructuredToken;
+
+ bool AddBearerToToken = false;
// This flag describes one's intention to connect managed database using secure or insecure sockets,
// but it will work only for certain kinds of databases.
- // For more details look through the parser implementations here (not all of them rely on this flag):
+ // For more details look through the parser implementations here (not all of them rely on this flag):
// https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r12426855#L229
- bool UseTls = false;
+ bool UseTls = false;
+
+ // For some of the data sources accessible via generic provider it's possible to specify the connection protocol.
+ // This setting may impact the throughput.
+ NConnector::NApi::EProtocol Protocol = NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED;
bool operator==(const TDatabaseAuth& other) const {
return std::tie(StructuredToken, AddBearerToToken, UseTls) == std::tie(other.StructuredToken, other.AddBearerToToken, other.UseTls);
@@ -80,7 +85,7 @@ struct TDatabaseResolverResponse {
TString Database;
bool Secure = false;
- TString ToDebugString() const {
+ TString ToDebugString() const {
return TStringBuilder() << "endpoint=" << Endpoint
<< ", host=" << Host
<< ", port=" << Port
diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h
index a5518bbdbe..abf38158aa 100644
--- a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h
+++ b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h
@@ -9,14 +9,19 @@ namespace NYql {
// https://cloud.yandex.ru/docs/managed-clickhouse/api-ref/Cluster/listHosts
class IMdbEndpointGenerator {
public:
+ struct TParams {
+ NYql::EDatabaseType DatabaseType;
+ TString MdbHost;
+ bool UseTls;
+ NConnector::NApi::EProtocol Protocol;
+ };
+
using TPtr = std::shared_ptr<IMdbEndpointGenerator>;
using TEndpoint = std::pair<TString, ui32>;
+
// Returns FQDN and port of the database host
- virtual TEndpoint ToEndpoint(
- const NYql::EDatabaseType databaseType,
- const TString& mdbHost,
- bool useTls) const = 0;
+ virtual TEndpoint ToEndpoint(const TParams& params) const = 0;
virtual ~IMdbEndpointGenerator() = default;
};
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
index ffc96887dc..ab3d16a0b5 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
@@ -26,6 +26,7 @@ func AnnotateLogger(logger log.Logger, method string, dsi *api_common.TDataSourc
log.UInt32("port", dsi.Endpoint.Port),
log.String("database", dsi.Database),
log.Bool("use_tls", dsi.UseTls),
+ log.String("protocol", dsi.Protocol.String()),
// TODO: can we print just a login without a password?
)
}
diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp
index 4b3c98ee53..0b4c93a8bf 100644
--- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp
+++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp
@@ -34,15 +34,10 @@ namespace NYql {
const TCredentials::TPtr& credentials) {
ValidateGenericClusterConfig(clusterConfig, "TGenericConfiguration::AddCluster");
+ YQL_CLOG(INFO, ProviderGeneric) << "generic provider add cluster: " << DumpGenericClusterConfig(clusterConfig);
+
const auto& clusterName = clusterConfig.GetName();
const auto& databaseId = clusterConfig.GetDatabaseId();
- const auto& endpoint = clusterConfig.GetEndpoint();
-
- YQL_CLOG(INFO, ProviderGeneric)
- << "generic provider add cluster"
- << ": name = " << clusterName
- << ", database id = " << databaseId
- << ", endpoint = " << endpoint;
if (databaseId) {
if (!databaseResolver) {
@@ -52,7 +47,11 @@ namespace NYql {
const auto token = MakeStructuredToken(clusterConfig, credentials);
databaseAuth[std::make_pair(databaseId, DatabaseTypeFromDataSourceKind(clusterConfig.GetKind()))] =
- NYql::TDatabaseAuth{token, /*AddBearer=*/true, clusterConfig.GetUseSsl()};
+ NYql::TDatabaseAuth{
+ .StructuredToken = token,
+ .AddBearerToToken = true,
+ .UseTls = clusterConfig.GetUseSsl(),
+ .Protocol = clusterConfig.GetProtocol()};
DatabaseIdsToClusterNames[databaseId].emplace_back(clusterName);
YQL_CLOG(DEBUG, ProviderGeneric) << "database id '" << databaseId << "' added to mapping";
@@ -96,6 +95,23 @@ namespace NYql {
"or set (ServiceAccountId && ServiceAccountIdSignature) in cluster config";
}
+ TString TGenericConfiguration::DumpGenericClusterConfig(const TGenericClusterConfig& clusterConfig) const {
+ TStringBuilder sb;
+ sb << "name = " << clusterConfig.GetName()
+ << ", kind = " << NConnector::NApi::EDataSourceKind_Name(clusterConfig.GetKind())
+ << ", database name = " << clusterConfig.GetDatabaseName()
+ << ", database id = " << clusterConfig.GetName()
+ << ", endpoint = " << clusterConfig.GetEndpoint()
+ << ", use tls = " << clusterConfig.GetUseSsl()
+ << ", protocol = " << NConnector::NApi::EProtocol_Name(clusterConfig.GetProtocol());
+
+ for (const auto& [key, value] : clusterConfig.GetDataSourceOptions()) {
+ sb << ", " << key << " = " << value;
+ }
+
+ return sb;
+ }
+
TGenericSettings::TConstPtr TGenericConfiguration::Snapshot() const {
return std::make_shared<const TGenericSettings>(*this);
}
diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h
index a58f4c31c3..d783963a65 100644
--- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h
+++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h
@@ -40,6 +40,7 @@ namespace NYql {
private:
TString MakeStructuredToken(const TGenericClusterConfig& clusterConfig, const TCredentials::TPtr& credentials) const;
+ TString DumpGenericClusterConfig(const TGenericClusterConfig& clusterConfig) const;
public:
THashMap<TString, TString> Tokens;