diff options
author | vitalyisaev <vitalyisaev@ydb.tech> | 2023-11-24 10:32:28 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@ydb.tech> | 2023-11-24 10:52:28 +0300 |
commit | d3e4beea087cbfdd703ef23136c307dd67ca9ae0 (patch) | |
tree | 7e897a1599d151f7b1d63e5e3b37eef95e8b4384 | |
parent | b04e3c5ba75a4d9130991f7e3ed55b037d9275cb (diff) | |
download | ydb-d3e4beea087cbfdd703ef23136c307dd67ca9ae0.tar.gz |
YQ Connector: use state of connection instead of service config when determining protocol
12 files changed, 208 insertions, 83 deletions
diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/actors/database_resolver.cpp index 8e3a50ea46..3ec5343da5 100644 --- a/ydb/core/fq/libs/actors/database_resolver.cpp +++ b/ydb/core/fq/libs/actors/database_resolver.cpp @@ -22,7 +22,12 @@ using namespace NActors; using namespace NYql; using TDatabaseDescription = NYql::TDatabaseResolverResponse::TDatabaseDescription; -using TParser = std::function<TDatabaseDescription(NJson::TJsonValue& body, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls)>; +using TParser = std::function<TDatabaseDescription( + NJson::TJsonValue& body, + const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, + bool useTls, + NConnector::NApi::EProtocol protocol +)>; using TParsers = THashMap<NYql::EDatabaseType, TParser>; struct TResolveParams { @@ -144,7 +149,11 @@ private: TParsers::const_iterator parserIt; if (parseJsonOk && (parserIt = Parsers.find(params.DatabaseType)) != Parsers.end()) { try { - auto description = parserIt->second(databaseInfo, MdbEndpointGenerator, params.DatabaseAuth.UseTls); + auto description = parserIt->second( + databaseInfo, + MdbEndpointGenerator, + params.DatabaseAuth.UseTls, + params.DatabaseAuth.Protocol); LOG_D("ResponseProcessor::Handle(HttpIncomingResponse): got description" << ": params: " << params.ToDebugString() << ", description: " << description.ToDebugString()); DatabaseId2Description[std::make_pair(params.Id, params.DatabaseType)] = description; @@ -223,7 +232,7 @@ public: .SetErrorTtl(TDuration::Minutes(1)) .SetMaxSize(1000000)) { - auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool) { + auto ydbParser = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr&, bool, NConnector::NApi::EProtocol) { bool secure = false; TString endpoint = databaseInfo.GetMap().at("endpoint").GetStringRobust(); TString prefix("/?database="); @@ -245,10 +254,14 @@ public: return TDatabaseDescription{endpoint, "", 0, database, secure}; }; Parsers[NYql::EDatabaseType::Ydb] = ydbParser; - Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) + Parsers[NYql::EDatabaseType::DataStreams] = [ydbParser]( + NJson::TJsonValue& databaseInfo, + const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, + bool useTls, + NConnector::NApi::EProtocol protocol) { bool isDedicatedDb = databaseInfo.GetMap().contains("storageConfig"); - auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls); + auto ret = ydbParser(databaseInfo, mdbEndpointGenerator, useTls, protocol); // TODO: Take explicit field from MVP if (!isDedicatedDb && ret.Endpoint.StartsWith("ydb.")) { // Replace "ydb." -> "yds." @@ -256,7 +269,12 @@ public: } return ret; }; - Parsers[NYql::EDatabaseType::ClickHouse] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) { + Parsers[NYql::EDatabaseType::ClickHouse] = []( + NJson::TJsonValue& databaseInfo, + const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, + bool useTls, + NConnector::NApi::EProtocol protocol + ) { NYql::IMdbEndpointGenerator::TEndpoint endpoint; TVector<TString> aliveHosts; @@ -270,16 +288,24 @@ public: ythrow yexception() << "No ALIVE ClickHouse hosts found"; } - endpoint = mdbEndpointGenerator->ToEndpoint( - NYql::EDatabaseType::ClickHouse, - aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], - useTls - ); + NYql::IMdbEndpointGenerator::TParams params = { + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], + .UseTls = useTls, + .Protocol = protocol, + }; + + endpoint = mdbEndpointGenerator->ToEndpoint(params); return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls}; }; - Parsers[NYql::EDatabaseType::PostgreSQL] = [](NJson::TJsonValue& databaseInfo, const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, bool useTls) { + Parsers[NYql::EDatabaseType::PostgreSQL] = []( + NJson::TJsonValue& databaseInfo, + const NYql::IMdbEndpointGenerator::TPtr& mdbEndpointGenerator, + bool useTls, + NConnector::NApi::EProtocol protocol + ) { NYql::IMdbEndpointGenerator::TEndpoint endpoint; TVector<TString> aliveHosts; @@ -302,11 +328,14 @@ public: ythrow yexception() << "No ALIVE PostgreSQL hosts found"; } - endpoint = mdbEndpointGenerator->ToEndpoint( - NYql::EDatabaseType::PostgreSQL, - aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], - useTls - ); + NYql::IMdbEndpointGenerator::TParams params = { + .DatabaseType = NYql::EDatabaseType::PostgreSQL, + .MdbHost = aliveHosts[std::rand() % static_cast<int>(aliveHosts.size())], + .UseTls = useTls, + .Protocol = protocol, + }; + + endpoint = mdbEndpointGenerator->ToEndpoint(params); return TDatabaseDescription{"", endpoint.first, endpoint.second, "", useTls}; }; diff --git a/ydb/core/fq/libs/compute/common/run_actor_params.cpp b/ydb/core/fq/libs/compute/common/run_actor_params.cpp index 048540c2ed..1fa989e1ac 100644 --- a/ydb/core/fq/libs/compute/common/run_actor_params.cpp +++ b/ydb/core/fq/libs/compute/common/run_actor_params.cpp @@ -73,7 +73,7 @@ TRunActorParams::TRunActorParams( , Scope(scope) , AuthToken(authToken) , DatabaseResolver(databaseResolver) - , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(config.GetCommon().GetMdbTransformHost(), config.GetCommon().GetUseNativeProtocolForClickHouse())) + , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(config.GetCommon().GetMdbTransformHost())) , QueryId(queryId) , UserId(userId) , Owner(owner) diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp index 88466ce081..b957a5052e 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.cpp @@ -18,15 +18,15 @@ namespace NFq { // TMdbEndpointGeneratorLegacy implements behavior required by YQL legacy ClickHouse provider class TMdbEndpointGeneratorLegacy: public NYql::IMdbEndpointGenerator { - TEndpoint ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override { + TEndpoint ToEndpoint(const NYql::IMdbEndpointGenerator::TParams& params) const override { // Inherited from here // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r11819335#L27 - if (databaseType == NYql::EDatabaseType::ClickHouse) { - auto port = useTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT; - return TEndpoint(ReplaceDomain(mdbHost), port); + if (params.DatabaseType == NYql::EDatabaseType::ClickHouse) { + auto port = params.UseTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT; + return TEndpoint(ReplaceDomain(params.MdbHost), port); } - ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); + ythrow yexception() << "Unexpected database type: " << ToString(params.DatabaseType); } }; @@ -39,48 +39,58 @@ namespace NFq { // that interacts with data sources through a separate Connector service class TMdbEndpointGeneratorGeneric: public NYql::IMdbEndpointGenerator { public: - TMdbEndpointGeneratorGeneric(bool transformHost, bool useNativeProtocolForClickHouse = false) - : TransformHost(transformHost), UseNativeProtocolForClickHouse(useNativeProtocolForClickHouse) + TMdbEndpointGeneratorGeneric(bool transformHost) + : TransformHost(transformHost) { } - TEndpoint ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost, bool useTls) const override { - auto fixedHost = TransformHost ? ReplaceDomain(mdbHost) : mdbHost; + TEndpoint ToEndpoint(const NYql::IMdbEndpointGenerator::TParams& params) const override { + auto fixedHost = TransformHost ? ReplaceDomain(params.MdbHost) : params.MdbHost; - switch (databaseType) { + switch (params.DatabaseType) { case NYql::EDatabaseType::ClickHouse: { // https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect ui32 port; - if (UseNativeProtocolForClickHouse) { - port = useTls ? CLICKHOUSE_NATIVE_SECURE_PORT : CLICKHOUSE_NATIVE_INSECURE_PORT; - } else { - port = useTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT; + + switch (params.Protocol) { + case NYql::NConnector::NApi::EProtocol::NATIVE: + port = params.UseTls ? CLICKHOUSE_NATIVE_SECURE_PORT : CLICKHOUSE_NATIVE_INSECURE_PORT; + break; + case NYql::NConnector::NApi::EProtocol::HTTP: + port = params.UseTls ? CLICKHOUSE_HTTP_SECURE_PORT : CLICKHOUSE_HTTP_INSECURE_PORT; + break; + default: + ythrow yexception() << "Unexpected protocol for ClickHouse: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); } return TEndpoint(fixedHost, port); } case NYql::EDatabaseType::PostgreSQL: // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect - return TEndpoint(fixedHost, POSTGRESQL_PORT); + switch (params.Protocol) { + case NYql::NConnector::NApi::EProtocol::NATIVE: + return TEndpoint(fixedHost, POSTGRESQL_PORT); + default: + ythrow yexception() << "Unexpected protocol for PostgreSQL: " << NYql::NConnector::NApi::EProtocol_Name(params.Protocol); + } default: - ythrow yexception() << TStringBuilder() << "Unexpected database type: " << int(databaseType); + ythrow yexception() << "Unexpected database type: " << ToString(params.DatabaseType); }; } private: bool TransformHost; - bool UseNativeProtocolForClickHouse; }; NYql::IMdbEndpointGenerator::TPtr - MakeMdbEndpointGeneratorGeneric(bool transformHost, bool useNativeProtocolForClickHouse) { - return std::make_shared<TMdbEndpointGeneratorGeneric>(transformHost, useNativeProtocolForClickHouse); + MakeMdbEndpointGeneratorGeneric(bool transformHost) { + return std::make_shared<TMdbEndpointGeneratorGeneric>(transformHost); } // TMdbEndpointGeneratorNoop just does nothing class TMdbEndpointGeneratorNoop: public NYql::IMdbEndpointGenerator { - TEndpoint ToEndpoint(const NYql::EDatabaseType, const TString& mdbHost, bool) const override { - return TEndpoint(mdbHost, 0); + TEndpoint ToEndpoint(const NYql::IMdbEndpointGenerator::TParams& params) const override { + return TEndpoint(params.MdbHost, 0); } }; @@ -88,4 +98,4 @@ namespace NFq { MakeMdbEndpointGeneratorNoop() { return std::make_shared<TMdbEndpointGeneratorNoop>(); } -}
\ No newline at end of file +} diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h index 75332332cb..f7be4f68de 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_endpoint_generator.h @@ -5,6 +5,6 @@ namespace NFq { NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorLegacy(); - NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorGeneric(bool transformHost, bool useNativeProtocolForClickHouse = false); + NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorGeneric(bool transformHost); NYql::IMdbEndpointGenerator::TPtr MakeMdbEndpointGeneratorNoop(); } diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp index 687570c5de..f02d3a4e08 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ut/mdb_endpoint_generator_ut.cpp @@ -7,48 +7,115 @@ Y_UNIT_TEST_SUITE(MdbEndpoingGenerator) { Y_UNIT_TEST(Legacy) { auto transformer = NFq::MakeMdbEndpointGeneratorLegacy(); + auto params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = "rc1c-p5waby2y5y1kb5ue.db.yandex.net", + .UseTls = true, + .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + }; + UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1c-p5waby2y5y1kb5ue.db.yandex.net", true), + transformer->ToEndpoint(params), TEndpoint("rc1c-p5waby2y5y1kb5ue.db.yandex.net", 8443)); + params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = "ya.ru", + .UseTls = false, + .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + }; + UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "ya.ru", false), + transformer->ToEndpoint(params), TEndpoint("ya.db.yandex.net", 8123)); } Y_UNIT_TEST(Generic_NoTransformHost) { auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(false); + auto params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", + .UseTls = true, + .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + }; + UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", true), + transformer->ToEndpoint(params), TEndpoint("rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", 8443)); + params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::PostgreSQL, + .MdbHost = "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", + .UseTls = false, + .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + }; + UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", false), + transformer->ToEndpoint(params), TEndpoint("rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", 6432)); } Y_UNIT_TEST(Generic_WithTransformHost) { auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(true); + // ClickHouse + + auto params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", + .UseTls = false, + .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + }; + UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", false), + transformer->ToEndpoint(params), TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 8123)); + params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", + .UseTls = false, + .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + }; + UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", true), - TEndpoint("rc1b-eyt6dtobu96rwydq.db.yandex.net", 6432)); - } + transformer->ToEndpoint(params), + TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 9000)); - Y_UNIT_TEST(Generic_ClickhouseNative) { - auto transformer = NFq::MakeMdbEndpointGeneratorGeneric(false, true); + params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", + .UseTls = true, + .Protocol = NYql::NConnector::NApi::EProtocol::HTTP, + }; UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", true), - TEndpoint("rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", 9440)); + transformer->ToEndpoint(params), + TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 8443)); + + params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::ClickHouse, + .MdbHost = "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net", + .UseTls = true, + .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + }; UNIT_ASSERT_VALUES_EQUAL( - transformer->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", false), - TEndpoint("rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", 9000)); + transformer->ToEndpoint(params), + TEndpoint("rc1a-d6dv17lv47v5mcop.db.yandex.net", 9440)); + + // PostgreSQL + + params = NYql::IMdbEndpointGenerator::TParams{ + .DatabaseType = NYql::EDatabaseType::PostgreSQL, + .MdbHost = "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net", + .UseTls = true, + .Protocol = NYql::NConnector::NApi::EProtocol::NATIVE, + }; + + UNIT_ASSERT_VALUES_EQUAL( + transformer->ToEndpoint(params), + TEndpoint("rc1b-eyt6dtobu96rwydq.db.yandex.net", 6432)); } } diff --git a/ydb/core/fq/libs/test_connection/test_connection.cpp b/ydb/core/fq/libs/test_connection/test_connection.cpp index fd67251891..521442ade4 100644 --- a/ydb/core/fq/libs/test_connection/test_connection.cpp +++ b/ydb/core/fq/libs/test_connection/test_connection.cpp @@ -132,7 +132,7 @@ public: , Counters(counters) , Signer(signer) , HttpGateway(httpGateway) - , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(commonConfig.GetMdbTransformHost(), commonConfig.GetUseNativeProtocolForClickHouse())) + , MdbEndpointGenerator(NFq::MakeMdbEndpointGeneratorGeneric(commonConfig.GetMdbTransformHost())) {} static constexpr char ActorName[] = "YQ_TEST_CONNECTION"; diff --git a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp index 345ea4b5ea..ea5004f832 100644 --- a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp +++ b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp @@ -75,16 +75,7 @@ namespace NKikimr::NKqp { ConnectorClient = NYql::NConnector::MakeClientGRPC(GenericGatewaysConfig.GetConnector()); if (queryServiceConfig.HasMdbTransformHost()) { - auto mdbTransformHost = queryServiceConfig.GetMdbTransformHost(); - bool useNativeProtocolForClickHouse = false; - for (const auto& p: queryServiceConfig.GetGeneric().GetDefaultSettings()) { - if (p.GetName() == "UseNativeProtocolForClickHouse") { - TryFromString<bool>(p.GetValue(), useNativeProtocolForClickHouse); - break; - } - } - - MdbEndpointGenerator = NFq::MakeMdbEndpointGeneratorGeneric(mdbTransformHost, useNativeProtocolForClickHouse); + MdbEndpointGenerator = NFq::MakeMdbEndpointGeneratorGeneric(queryServiceConfig.GetMdbTransformHost()); } // Create actors required for MDB database resolving diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h index 0e2fc958c6..82bdb4f0ca 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h +++ b/ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h @@ -53,14 +53,19 @@ inline TString DatabaseTypeToMdbUrlPath(EDatabaseType databaseType) { struct TDatabaseAuth { // Serialized token value used to access MDB API - TString StructuredToken; - bool AddBearerToToken = false; + TString StructuredToken; + + bool AddBearerToToken = false; // This flag describes one's intention to connect managed database using secure or insecure sockets, // but it will work only for certain kinds of databases. - // For more details look through the parser implementations here (not all of them rely on this flag): + // For more details look through the parser implementations here (not all of them rely on this flag): // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/actors/database_resolver.cpp?rev=r12426855#L229 - bool UseTls = false; + bool UseTls = false; + + // For some of the data sources accessible via generic provider it's possible to specify the connection protocol. + // This setting may impact the throughput. + NConnector::NApi::EProtocol Protocol = NConnector::NApi::EProtocol::PROTOCOL_UNSPECIFIED; bool operator==(const TDatabaseAuth& other) const { return std::tie(StructuredToken, AddBearerToToken, UseTls) == std::tie(other.StructuredToken, other.AddBearerToToken, other.UseTls); @@ -80,7 +85,7 @@ struct TDatabaseResolverResponse { TString Database; bool Secure = false; - TString ToDebugString() const { + TString ToDebugString() const { return TStringBuilder() << "endpoint=" << Endpoint << ", host=" << Host << ", port=" << Port diff --git a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h index a5518bbdbe..abf38158aa 100644 --- a/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h +++ b/ydb/library/yql/providers/common/db_id_async_resolver/mdb_endpoint_generator.h @@ -9,14 +9,19 @@ namespace NYql { // https://cloud.yandex.ru/docs/managed-clickhouse/api-ref/Cluster/listHosts class IMdbEndpointGenerator { public: + struct TParams { + NYql::EDatabaseType DatabaseType; + TString MdbHost; + bool UseTls; + NConnector::NApi::EProtocol Protocol; + }; + using TPtr = std::shared_ptr<IMdbEndpointGenerator>; using TEndpoint = std::pair<TString, ui32>; + // Returns FQDN and port of the database host - virtual TEndpoint ToEndpoint( - const NYql::EDatabaseType databaseType, - const TString& mdbHost, - bool useTls) const = 0; + virtual TEndpoint ToEndpoint(const TParams& params) const = 0; virtual ~IMdbEndpointGenerator() = default; }; diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go index ffc96887dc..ab3d16a0b5 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go +++ b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go @@ -26,6 +26,7 @@ func AnnotateLogger(logger log.Logger, method string, dsi *api_common.TDataSourc log.UInt32("port", dsi.Endpoint.Port), log.String("database", dsi.Database), log.Bool("use_tls", dsi.UseTls), + log.String("protocol", dsi.Protocol.String()), // TODO: can we print just a login without a password? ) } diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp index 4b3c98ee53..0b4c93a8bf 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp @@ -34,15 +34,10 @@ namespace NYql { const TCredentials::TPtr& credentials) { ValidateGenericClusterConfig(clusterConfig, "TGenericConfiguration::AddCluster"); + YQL_CLOG(INFO, ProviderGeneric) << "generic provider add cluster: " << DumpGenericClusterConfig(clusterConfig); + const auto& clusterName = clusterConfig.GetName(); const auto& databaseId = clusterConfig.GetDatabaseId(); - const auto& endpoint = clusterConfig.GetEndpoint(); - - YQL_CLOG(INFO, ProviderGeneric) - << "generic provider add cluster" - << ": name = " << clusterName - << ", database id = " << databaseId - << ", endpoint = " << endpoint; if (databaseId) { if (!databaseResolver) { @@ -52,7 +47,11 @@ namespace NYql { const auto token = MakeStructuredToken(clusterConfig, credentials); databaseAuth[std::make_pair(databaseId, DatabaseTypeFromDataSourceKind(clusterConfig.GetKind()))] = - NYql::TDatabaseAuth{token, /*AddBearer=*/true, clusterConfig.GetUseSsl()}; + NYql::TDatabaseAuth{ + .StructuredToken = token, + .AddBearerToToken = true, + .UseTls = clusterConfig.GetUseSsl(), + .Protocol = clusterConfig.GetProtocol()}; DatabaseIdsToClusterNames[databaseId].emplace_back(clusterName); YQL_CLOG(DEBUG, ProviderGeneric) << "database id '" << databaseId << "' added to mapping"; @@ -96,6 +95,23 @@ namespace NYql { "or set (ServiceAccountId && ServiceAccountIdSignature) in cluster config"; } + TString TGenericConfiguration::DumpGenericClusterConfig(const TGenericClusterConfig& clusterConfig) const { + TStringBuilder sb; + sb << "name = " << clusterConfig.GetName() + << ", kind = " << NConnector::NApi::EDataSourceKind_Name(clusterConfig.GetKind()) + << ", database name = " << clusterConfig.GetDatabaseName() + << ", database id = " << clusterConfig.GetName() + << ", endpoint = " << clusterConfig.GetEndpoint() + << ", use tls = " << clusterConfig.GetUseSsl() + << ", protocol = " << NConnector::NApi::EProtocol_Name(clusterConfig.GetProtocol()); + + for (const auto& [key, value] : clusterConfig.GetDataSourceOptions()) { + sb << ", " << key << " = " << value; + } + + return sb; + } + TGenericSettings::TConstPtr TGenericConfiguration::Snapshot() const { return std::make_shared<const TGenericSettings>(*this); } diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h index a58f4c31c3..d783963a65 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h +++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h @@ -40,6 +40,7 @@ namespace NYql { private: TString MakeStructuredToken(const TGenericClusterConfig& clusterConfig, const TCredentials::TPtr& credentials) const; + TString DumpGenericClusterConfig(const TGenericClusterConfig& clusterConfig) const; public: THashMap<TString, TString> Tokens; |