YQ Connector: optionally enable TLS connections between Connector and the data source

Теперь клиент сервиса Коннектора может указывать, должен ли он организовать защищённое или обычное соединение с источником данных. YQ всегда будет требовать TLS между Коннектором и источниками.

8 files changed, 26 insertions, 8 deletions

diff --git a/ydb/core/fq/libs/actors/clusters_from_connections.cpp b/ydb/core/fq/libs/actors/clusters_from_connections.cpp
index 8b734d81aae..db185ba84d0 100644
--- a/ydb/core/fq/libs/actors/clusters_from_connections.cpp
+++ b/ydb/core/fq/libs/actors/clusters_from_connections.cpp
@@ -102,6 +102,10 @@ void FillGenericClusterConfig(
         clusterCfg.mutable_credentials()->mutable_basic()->set_username(connection.login());
         clusterCfg.mutable_credentials()->mutable_basic()->set_password(connection.password());
         FillClusterAuth(clusterCfg, connection.auth(), authToken, accountIdSignatures);
+        // Since resolver always returns secure ports, we'll always ask for secure connections
+        // between remote Connector and the data source:
+        // https://a.yandex-team.ru/arcadia/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp#L24
+        clusterCfg.SetUseSsl(true);
 }
 
 } //namespace
diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
index 8772a7bf131..784405d050c 100644
--- a/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
+++ b/ydb/core/fq/libs/db_id_async_resolver_impl/mdb_host_transformer.cpp
@@ -25,8 +25,9 @@ namespace NFq {
         TString ToEndpoint(const NYql::EDatabaseType databaseType, const TString& mdbHost) const override {
             switch (databaseType) {
                 case NYql::EDatabaseType::ClickHouse:
-                    // TODO: https://st.yandex-team.ru/YQ-2170: support secure connections on 9440
-                    return mdbHost + ":9000";
+                    // https://cloud.yandex.ru/docs/managed-clickhouse/operations/connect
+                    // TODO: fix Native protocol + TLS https://st.yandex-team.ru/YQ-2286
+                    return mdbHost + ":8443";
                 case NYql::EDatabaseType::PostgreSQL:
                     // https://cloud.yandex.ru/docs/managed-postgresql/operations/connect
                     return mdbHost + ":6432";
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 4fec8c450e6..0ffd18007e6 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -568,6 +568,10 @@ message TGenericClusterConfig {
     optional string ServiceAccountIdSignature = 7;
     optional string Token = 11;
 
+    // If true, the generic provider will ask connector server to use secure connections 
+    // to access remote data sources.
+    optional bool UseSsl = 12;
+
     reserved 2, 3, 5;
 }
 
@@ -576,9 +580,9 @@ message TGenericConnectorConfig {
     required NYql.NConnector.NApi.TEndpoint Endpoint = 3;
     // If true, GRPC Client will use TLS encryption.
     // Server cert will be verified with system CA cert pool.
-    required bool UseTLS = 2;
+    required bool UseSsl = 4;
 
-    reserved 1;
+    reserved 1, 2;
 }
 
 message TGenericGatewayConfig {
diff --git a/ydb/library/yql/providers/generic/connector/api/common/data_source.proto b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
index 7218c22c7bb..f9da3d4de0d 100644
--- a/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
+++ b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
@@ -26,10 +26,17 @@ enum EDataSourceKind {
     POSTGRESQL = 2;
 }
 
-// TDataSourceInstance helps to identify the instance of a data source to route request to.
+// TDataSourceInstance helps to identify the instance of a data source to redirect request to.
 message TDataSourceInstance {
+    // Data source kind
     EDataSourceKind kind = 1;
+    // Network address
     TEndpoint endpoint = 2;
+    // Database name
     string database = 3;
+    // Credentials to access database
     TCredentials credentials = 4;
+    // If true, Connector server will use secure connections to access remote data sources.
+    // Certificates will be obtained from the standard system paths.
+    bool use_tls = 5;
 }
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/cli/main.cpp b/ydb/library/yql/providers/generic/connector/libcpp/cli/main.cpp
index 71dfa22f1d4..eb5777a27fe 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/cli/main.cpp
+++ b/ydb/library/yql/providers/generic/connector/libcpp/cli/main.cpp
@@ -92,7 +92,7 @@ int main() {
     NYql::TGenericConnectorConfig cfg;
     cfg.mutable_endpoint()->set_host("connector.yql-streaming.cloud.yandex.net");
     cfg.mutable_endpoint()->set_port(50051);
-    cfg.SetUseTLS(true);
+    cfg.SetUseSsl(true);
 
     auto client = NYql::NConnector::MakeClientGRPC(cfg);
 
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp b/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
index c1d69f5552e..85d037a3922 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
+++ b/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
@@ -26,7 +26,7 @@ namespace NYql::NConnector {
         std::shared_ptr<grpc::ChannelCredentials> credentials;
         auto networkEndpoint = cfg.GetEndpoint().host() + ":" + std::to_string(cfg.GetEndpoint().port());
 
-        if (cfg.GetUseTLS()) {
+        if (cfg.GetUseSsl()) {
             // Hopefully GRPC will find appropriate CA cert in system folders
             credentials = grpc::SslCredentials(grpc::SslCredentialsOptions());
         } else {
diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp
index bc53eb51e43..a7416e27c6e 100644
--- a/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp
+++ b/ydb/library/yql/providers/generic/provider/yql_generic_load_meta.cpp
@@ -124,6 +124,8 @@ namespace NYql {
                 dsi->set_database(TString(db));
                 request.set_table(TString(dbTable));
 
+                dsi->set_use_tls(clusterConfig.GetUseSsl());
+
                 // NOTE: errors will be checked further in DoApplyAsyncChanges
                 Results_.emplace(item, TGenericTableDescription(request.data_source_instance(), Client_->DescribeTable(request)));
 
diff --git a/ydb/services/fq/ut_integration/fq_ut.cpp b/ydb/services/fq/ut_integration/fq_ut.cpp
index a13fd6f13b5..b80e4b6f9fd 100644
--- a/ydb/services/fq/ut_integration/fq_ut.cpp
+++ b/ydb/services/fq/ut_integration/fq_ut.cpp
@@ -870,7 +870,7 @@ Y_UNIT_TEST_SUITE(Yq_2) {
         {
             auto transformer = ::NFq::MakeTMdbHostTransformerGeneric();
             UNIT_ASSERT_VALUES_EQUAL(::NFq::MakeTMdbHostTransformerGeneric()->ToEndpoint(NYql::EDatabaseType::ClickHouse, "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"),
-                                    "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net:9000");
+                                    "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net:8443");
             UNIT_ASSERT_VALUES_EQUAL(::NFq::MakeTMdbHostTransformerGeneric()->ToEndpoint(NYql::EDatabaseType::PostgreSQL, "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net"),
                                     "rc1b-eyt6dtobu96rwydq.mdb.yandexcloud.net:6432");
         }