aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvitalyisaev <vitalyisaev@yandex-team.com>2023-06-19 16:09:10 +0300
committervitalyisaev <vitalyisaev@yandex-team.com>2023-06-19 16:09:10 +0300
commitc7fa117ab3122dd361bcc50dd975c12bb6714bbb (patch)
tree3eda0ae81f35ad70686dbf99b6f9fcc2cd1ce6f7
parent128fa513a8669e87b8e2322efa61c7b82a8c085d (diff)
downloadydb-c7fa117ab3122dd361bcc50dd975c12bb6714bbb.tar.gz
Prepare Connector for deployment in Cloud
1. В сервер добавлен режим работы с TLS и протестирован с помощью клиента на самоподписных сертификатах. 2. Конфигурации клиента и сервера теперь описаны в Protobuf, при этом они переиспользуют некоторые типы из API Коннектора. 3. В связи с п. 2 API отрефакторено и разделено по двум папкам: `ydb/library/yql/providers/generic/connector/api/common` и `ydb/library/yql/providers/generic/connector/api/service` 4. Добавлена большая часть скриптов для сборки через `ya package` и разворачивания сервиса (по аналогии с `yq-watchdog`).
-rw-r--r--library/go/core/buildinfo/test/main.go15
-rw-r--r--library/go/core/buildinfo/test/ya.make5
-rw-r--r--tools/go_test_miner/main.go168
-rw-r--r--tools/go_test_miner/ya.make16
-rw-r--r--ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt2
-rw-r--r--ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt2
-rw-r--r--ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt2
-rw-r--r--ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt2
-rw-r--r--ydb/library/yql/providers/common/proto/gateways_config.proto4
-rw-r--r--ydb/library/yql/providers/common/proto/ya.make2
-rw-r--r--ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp2
-rw-r--r--ydb/library/yql/providers/generic/connector/api/CMakeLists.txt11
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/CMakeLists.darwin-x86_64.txt56
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-aarch64.txt57
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-x86_64.txt57
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/CMakeLists.txt (renamed from ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.txt)0
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/CMakeLists.windows-x86_64.txt56
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/data_source.proto39
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/endpoint.proto11
-rw-r--r--ydb/library/yql/providers/generic/connector/api/common/ya.make11
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/CMakeLists.darwin-x86_64.txt (renamed from ydb/library/yql/providers/generic/connector/api/CMakeLists.darwin-x86_64.txt)18
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-aarch64.txt (renamed from ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-aarch64.txt)18
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-x86_64.txt (renamed from ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-x86_64.txt)18
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/CMakeLists.windows-x86_64.txt (renamed from ydb/library/yql/providers/generic/connector/api/CMakeLists.windows-x86_64.txt)18
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/connector.proto (renamed from ydb/library/yql/providers/generic/connector/api/connector.proto)2
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.darwin-x86_64.txt (renamed from ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.darwin-x86_64.txt)13
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-aarch64.txt (renamed from ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-aarch64.txt)13
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-x86_64.txt (renamed from ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-x86_64.txt)13
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.windows-x86_64.txt (renamed from ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.windows-x86_64.txt)13
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto (renamed from ydb/library/yql/providers/generic/connector/api/protos/connector.proto)40
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/protos/ya.make (renamed from ydb/library/yql/providers/generic/connector/api/protos/ya.make)1
-rw-r--r--ydb/library/yql/providers/generic/connector/api/service/ya.make16
-rw-r--r--ydb/library/yql/providers/generic/connector/api/ya.make18
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt3
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt3
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt3
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt3
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/client.h4
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp2
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/error.h2
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/utils.h2
-rw-r--r--ydb/library/yql/providers/generic/connector/libcpp/ya.make3
-rw-r--r--ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt3
-rw-r--r--ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt3
-rw-r--r--ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt3
-rw-r--r--ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt3
-rw-r--r--ydb/library/yql/providers/generic/proto/source.proto3
-rw-r--r--ydb/library/yql/providers/generic/proto/ya.make3
-rw-r--r--ydb/library/yql/providers/generic/provider/yql_generic_settings.h2
51 files changed, 650 insertions, 148 deletions
diff --git a/library/go/core/buildinfo/test/main.go b/library/go/core/buildinfo/test/main.go
new file mode 100644
index 0000000000..a17ac12749
--- /dev/null
+++ b/library/go/core/buildinfo/test/main.go
@@ -0,0 +1,15 @@
+package main
+
+import (
+ "fmt"
+
+ "a.yandex-team.ru/library/go/core/buildinfo"
+)
+
+func main() {
+ if buildinfo.Info.ProgramVersion != "" {
+ fmt.Print(buildinfo.Info.ProgramVersion)
+ } else {
+ fmt.Printf("ProgramVersion is not available\n")
+ }
+}
diff --git a/library/go/core/buildinfo/test/ya.make b/library/go/core/buildinfo/test/ya.make
new file mode 100644
index 0000000000..6a0765382d
--- /dev/null
+++ b/library/go/core/buildinfo/test/ya.make
@@ -0,0 +1,5 @@
+GO_PROGRAM()
+
+SRCS(main.go)
+
+END()
diff --git a/tools/go_test_miner/main.go b/tools/go_test_miner/main.go
new file mode 100644
index 0000000000..43b729572e
--- /dev/null
+++ b/tools/go_test_miner/main.go
@@ -0,0 +1,168 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "go/importer"
+ "go/token"
+ "go/types"
+ "os"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "sort"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+const (
+ usageTemplate = "Usage: %s [-benchmarks] [-examples] [-tests] import-path\n"
+)
+
+func findObjectByName(pkg *types.Package, re *regexp.Regexp, name string) types.Object {
+ if pkg != nil && re != nil && len(name) > 0 {
+ if obj := pkg.Scope().Lookup(name); obj != nil {
+ if re.MatchString(obj.Type().String()) {
+ return obj
+ }
+ }
+ }
+ return nil
+}
+
+func isTestName(name, prefix string) bool {
+ ok := false
+ if strings.HasPrefix(name, prefix) {
+ if len(name) == len(prefix) {
+ ok = true
+ } else {
+ rune, _ := utf8.DecodeRuneInString(name[len(prefix):])
+ ok = !unicode.IsLower(rune)
+ }
+ }
+ return ok
+}
+
+func main() {
+ testsPtr := flag.Bool("tests", false, "report tests")
+ benchmarksPtr := flag.Bool("benchmarks", false, "report benchmarks")
+ examplesPtr := flag.Bool("examples", false, "report examples")
+
+ flag.Usage = func() {
+ _, _ = fmt.Fprintf(flag.CommandLine.Output(), usageTemplate, filepath.Base(os.Args[0]))
+ flag.PrintDefaults()
+ }
+
+ flag.Parse()
+
+ // Check if the number of positional parameters matches
+ args := flag.Args()
+ argsCount := len(args)
+ if argsCount != 1 {
+ exitCode := 0
+ if argsCount > 1 {
+ fmt.Println("Error: invalid number of parameters...")
+ exitCode = 1
+ }
+ flag.Usage()
+ os.Exit(exitCode)
+ }
+
+ importPath := args[0]
+
+ var fset token.FileSet
+ imp := importer.ForCompiler(&fset, runtime.Compiler, nil)
+ pkg, err := imp.Import(importPath)
+ if err != nil {
+ fmt.Printf("Error: %v\n", err)
+ os.Exit(1)
+ }
+
+ if !*testsPtr && !*benchmarksPtr && !*examplesPtr {
+ // Nothing to do, just exit normally
+ os.Exit(0)
+ }
+
+ // // First approach: just dump the package scope as a string
+ // // package "junk/snermolaev/libmath" scope 0xc0000df540 {
+ // // . func junk/snermolaev/libmath.Abs(a int) int
+ // // . func junk/snermolaev/libmath.AbsReport(s string)
+ // // . func junk/snermolaev/libmath.Sum(a int, b int) int
+ // // . func junk/snermolaev/libmath.TestAbs(t *testing.T)
+ // // . func junk/snermolaev/libmath.TestSum(t *testing.T)
+ // // . func junk/snermolaev/libmath.init()
+ // // }
+ // // and then collect all functions that match test function signature
+ // pkgPath := pkg.Path()
+ // scopeContent := strings.Split(pkg.Scope().String(), "\n")
+ // re := regexp.MustCompile("^\\.\\s*func\\s*" + pkgPath + "\\.(Test\\w*)\\(\\s*\\w*\\s*\\*\\s*testing\\.T\\s*\\)$")
+ // for _, name := range scopeContent {
+ // match := re.FindAllStringSubmatch(name, -1)
+ // if len(match) > 0 {
+ // fmt.Println(match[0][1])
+ // }
+ // }
+
+ // Second approach: look through all names defined in the pkg scope
+ // and collect those functions that match test function signature
+ // Unfortunately I failed to employ reflection mechinary for signature
+ // comparison for unknown reasons (this needs additional investigation
+ // I am going to use regexp as workaround for a while)
+ // testFunc := func (*testing.T) {}
+ // for ...
+ // ...
+ // if reflect.DeepEqual(obj.Type(), reflect.TypeOf(testFunc)) {
+ // // this condition doesn't work
+ // }
+ reBenchmark := regexp.MustCompile(`^func\(\w*\s*\*testing\.B\)$`)
+ reExample := regexp.MustCompile(`^func\(\s*\)$`)
+ reTest := regexp.MustCompile(`^func\(\w*\s*\*testing\.T\)$`)
+ reTestMain := regexp.MustCompile(`^func\(\w*\s*\*testing\.M\)$`)
+
+ var re *regexp.Regexp
+ names := pkg.Scope().Names()
+
+ var testFns []types.Object
+
+ for _, name := range names {
+ if name == "TestMain" && findObjectByName(pkg, reTestMain, name) != nil {
+ fmt.Println("#TestMain")
+ continue
+ }
+
+ switch {
+ case *benchmarksPtr && isTestName(name, "Benchmark"):
+ re = reBenchmark
+ case *examplesPtr && isTestName(name, "Example"):
+ re = reExample
+ case *testsPtr && isTestName(name, "Test"):
+ re = reTest
+ default:
+ continue
+ }
+
+ if obj := findObjectByName(pkg, re, name); obj != nil {
+ testFns = append(testFns, obj)
+ }
+ }
+
+ sort.Slice(testFns, func(i, j int) bool {
+ iPos := testFns[i].Pos()
+ jPos := testFns[j].Pos()
+
+ if !iPos.IsValid() || !jPos.IsValid() {
+ return iPos < jPos
+ }
+
+ iPosition := fset.PositionFor(iPos, true)
+ jPosition := fset.PositionFor(jPos, true)
+
+ return iPosition.Filename < jPosition.Filename ||
+ (iPosition.Filename == jPosition.Filename && iPosition.Line < jPosition.Line)
+ })
+
+ for _, testFn := range testFns {
+ fmt.Println(testFn.Name())
+ }
+}
diff --git a/tools/go_test_miner/ya.make b/tools/go_test_miner/ya.make
new file mode 100644
index 0000000000..2efea0ec7b
--- /dev/null
+++ b/tools/go_test_miner/ya.make
@@ -0,0 +1,16 @@
+GO_PROGRAM()
+
+SRCS(
+ main.go
+)
+
+GO_TEST_SRCS(
+ main_test.go
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ gotest
+)
+
diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt
index 0841a72dae..962067cd49 100644
--- a/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt
@@ -36,7 +36,7 @@ target_link_libraries(providers-common-proto PUBLIC
contrib-libs-cxxsupp
yutil
library-yql-protos
- connector-api-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-common-proto PRIVATE
diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt
index 8dade7c78c..9925b68fbf 100644
--- a/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt
@@ -37,7 +37,7 @@ target_link_libraries(providers-common-proto PUBLIC
contrib-libs-cxxsupp
yutil
library-yql-protos
- connector-api-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-common-proto PRIVATE
diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt
index 8dade7c78c..9925b68fbf 100644
--- a/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt
@@ -37,7 +37,7 @@ target_link_libraries(providers-common-proto PUBLIC
contrib-libs-cxxsupp
yutil
library-yql-protos
- connector-api-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-common-proto PRIVATE
diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt
index 0841a72dae..962067cd49 100644
--- a/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt
@@ -36,7 +36,7 @@ target_link_libraries(providers-common-proto PUBLIC
contrib-libs-cxxsupp
yutil
library-yql-protos
- connector-api-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-common-proto PRIVATE
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 24c82e2dc7..1a96f12b17 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -2,7 +2,8 @@ package NYql;
option java_package = "ru.yandex.yql.proto";
import "ydb/library/yql/protos/clickhouse.proto";
-import "ydb/library/yql/providers/generic/connector/api/protos/connector.proto";
+import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto";
+import "ydb/library/yql/providers/generic/connector/api/common/endpoint.proto";
/////////////////////////////// common ///////////////////////////////
@@ -556,6 +557,7 @@ message TGenericClusterConfig {
}
message TGenericGatewayConfig {
+ // TODO: replace with map<DataSourceKind, Endpoint>
required string Endpoint = 1;
reserved 2;
repeated TGenericClusterConfig ClusterMapping = 3;
diff --git a/ydb/library/yql/providers/common/proto/ya.make b/ydb/library/yql/providers/common/proto/ya.make
index 3c3ffac95d..fded415f56 100644
--- a/ydb/library/yql/providers/common/proto/ya.make
+++ b/ydb/library/yql/providers/common/proto/ya.make
@@ -7,7 +7,7 @@ SRCS(
PEERDIR(
ydb/library/yql/protos
- ydb/library/yql/providers/generic/connector/api/protos
+ ydb/library/yql/providers/generic/connector/api/common
)
IF (NOT PY_PROTOS_FOR)
diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp
index 8e49d47c74..70345916a6 100644
--- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp
+++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp
@@ -12,7 +12,7 @@
#include <ydb/library/yql/public/udf/arrow/util.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/yql_panic.h>
-#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h>
+#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h>
#include <ydb/library/yql/providers/generic/connector/libcpp/error.h>
namespace NYql::NDq {
diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt
index f8b31df0c1..459e4be130 100644
--- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt
+++ b/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt
@@ -6,12 +6,5 @@
# original buildsystem will not be accepted.
-if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
- include(CMakeLists.linux-aarch64.txt)
-elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
- include(CMakeLists.darwin-x86_64.txt)
-elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
- include(CMakeLists.windows-x86_64.txt)
-elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
- include(CMakeLists.linux-x86_64.txt)
-endif()
+add_subdirectory(common)
+add_subdirectory(service)
diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 0000000000..1af4185e07
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,56 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(connector-api-common)
+target_link_libraries(connector-api-common PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(connector-api-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto
+)
+target_proto_addincls(connector-api-common
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(connector-api-common
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..ca49587737
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,57 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(connector-api-common)
+target_link_libraries(connector-api-common PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(connector-api-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto
+)
+target_proto_addincls(connector-api-common
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(connector-api-common
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-x86_64.txt
new file mode 100644
index 0000000000..ca49587737
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,57 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(connector-api-common)
+target_link_libraries(connector-api-common PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(connector-api-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto
+)
+target_proto_addincls(connector-api-common
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(connector-api-common
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.txt
index f8b31df0c1..f8b31df0c1 100644
--- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.txt
+++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.txt
diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.windows-x86_64.txt
new file mode 100644
index 0000000000..1af4185e07
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,56 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+get_built_tool_path(
+ TOOL_protoc_bin
+ TOOL_protoc_dependency
+ contrib/tools/protoc/bin
+ protoc
+)
+get_built_tool_path(
+ TOOL_cpp_styleguide_bin
+ TOOL_cpp_styleguide_dependency
+ contrib/tools/protoc/plugins/cpp_styleguide
+ cpp_styleguide
+)
+
+add_library(connector-api-common)
+target_link_libraries(connector-api-common PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ contrib-libs-protobuf
+)
+target_proto_messages(connector-api-common PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto
+)
+target_proto_addincls(connector-api-common
+ ./
+ ${CMAKE_SOURCE_DIR}/
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
+)
+target_proto_outs(connector-api-common
+ --cpp_out=${CMAKE_BINARY_DIR}/
+ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/
+)
diff --git a/ydb/library/yql/providers/generic/connector/api/common/data_source.proto b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
new file mode 100644
index 0000000000..5c64d09711
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto
@@ -0,0 +1,39 @@
+syntax = "proto3";
+
+package NYql.Connector.API;
+
+import "ydb/library/yql/providers/generic/connector/api/common/endpoint.proto";
+
+option go_package = "a.yandex-team.ru/ydb/library/yql/providers/generic/connector/api/common";
+
+// Credentials represents various ways of user authentication in the data source instance
+message Credentials {
+ // Basic
+ message Basic {
+ string username = 1;
+ string password = 2;
+ }
+
+ oneof payload {
+ Basic basic = 1;
+ }
+}
+
+// DataSourceKind enumerates the external data sources
+// supported in the federated query system
+enum DataSourceKind {
+ DATA_SOURCE_KIND_RESERVED = 0;
+ CLICKHOUSE = 1;
+ POSTGRESQL = 2;
+}
+
+// DataSourceInstance helps to identify the instance of a data source to route request to.
+message DataSourceInstance {
+ DataSourceKind kind = 1;
+ Endpoint endpoint = 2;
+
+ // database name as it is
+ string database = 3;
+
+ Credentials credentials = 4;
+}
diff --git a/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto b/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto
new file mode 100644
index 0000000000..5b8ce1f2bd
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto
@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+package NYql.Connector.API;
+
+option go_package = "a.yandex-team.ru/ydb/library/yql/providers/generic/connector/api/common";
+
+// Endpoint represents the network address of a data source instance
+message Endpoint {
+ string host = 1;
+ uint32 port = 2;
+} \ No newline at end of file
diff --git a/ydb/library/yql/providers/generic/connector/api/common/ya.make b/ydb/library/yql/providers/generic/connector/api/common/ya.make
new file mode 100644
index 0000000000..0dd3d856cb
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/common/ya.make
@@ -0,0 +1,11 @@
+# WARNING: never add dependency to YDB protofiles here,
+# because it would break Go code generation.
+
+PROTO_LIBRARY()
+
+SRCS(
+ data_source.proto
+ endpoint.proto
+)
+
+END()
diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.darwin-x86_64.txt
index fc57461fef..75d63720a6 100644
--- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.darwin-x86_64.txt
@@ -26,21 +26,21 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(generic-connector-api)
-set_property(TARGET generic-connector-api PROPERTY
+add_library(connector-api-service)
+set_property(TARGET connector-api-service PROPERTY
PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
)
-target_link_libraries(generic-connector-api PUBLIC
+target_link_libraries(connector-api-service PUBLIC
contrib-libs-cxxsupp
yutil
contrib-libs-grpc
- connector-api-protos
+ api-service-protos
contrib-libs-protobuf
)
-target_proto_messages(generic-connector-api PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto
+target_proto_messages(connector-api-service PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto
)
-target_proto_addincls(generic-connector-api
+target_proto_addincls(connector-api-service
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -49,11 +49,11 @@ target_proto_addincls(generic-connector-api
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(generic-connector-api
+target_proto_outs(connector-api-service
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
-target_proto_plugin(generic-connector-api
+target_proto_plugin(connector-api-service
grpc_cpp
grpc_cpp
)
diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-aarch64.txt
index 1e9bb96043..b41c8ae098 100644
--- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-aarch64.txt
@@ -26,22 +26,22 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(generic-connector-api)
-set_property(TARGET generic-connector-api PROPERTY
+add_library(connector-api-service)
+set_property(TARGET connector-api-service PROPERTY
PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
)
-target_link_libraries(generic-connector-api PUBLIC
+target_link_libraries(connector-api-service PUBLIC
contrib-libs-linux-headers
contrib-libs-cxxsupp
yutil
contrib-libs-grpc
- connector-api-protos
+ api-service-protos
contrib-libs-protobuf
)
-target_proto_messages(generic-connector-api PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto
+target_proto_messages(connector-api-service PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto
)
-target_proto_addincls(generic-connector-api
+target_proto_addincls(connector-api-service
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -50,11 +50,11 @@ target_proto_addincls(generic-connector-api
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(generic-connector-api
+target_proto_outs(connector-api-service
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
-target_proto_plugin(generic-connector-api
+target_proto_plugin(connector-api-service
grpc_cpp
grpc_cpp
)
diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-x86_64.txt
index 1e9bb96043..b41c8ae098 100644
--- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-x86_64.txt
@@ -26,22 +26,22 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(generic-connector-api)
-set_property(TARGET generic-connector-api PROPERTY
+add_library(connector-api-service)
+set_property(TARGET connector-api-service PROPERTY
PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
)
-target_link_libraries(generic-connector-api PUBLIC
+target_link_libraries(connector-api-service PUBLIC
contrib-libs-linux-headers
contrib-libs-cxxsupp
yutil
contrib-libs-grpc
- connector-api-protos
+ api-service-protos
contrib-libs-protobuf
)
-target_proto_messages(generic-connector-api PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto
+target_proto_messages(connector-api-service PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto
)
-target_proto_addincls(generic-connector-api
+target_proto_addincls(connector-api-service
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -50,11 +50,11 @@ target_proto_addincls(generic-connector-api
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(generic-connector-api
+target_proto_outs(connector-api-service
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
-target_proto_plugin(generic-connector-api
+target_proto_plugin(connector-api-service
grpc_cpp
grpc_cpp
)
diff --git a/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.windows-x86_64.txt
index fc57461fef..75d63720a6 100644
--- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.windows-x86_64.txt
@@ -26,21 +26,21 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(generic-connector-api)
-set_property(TARGET generic-connector-api PROPERTY
+add_library(connector-api-service)
+set_property(TARGET connector-api-service PROPERTY
PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h
)
-target_link_libraries(generic-connector-api PUBLIC
+target_link_libraries(connector-api-service PUBLIC
contrib-libs-cxxsupp
yutil
contrib-libs-grpc
- connector-api-protos
+ api-service-protos
contrib-libs-protobuf
)
-target_proto_messages(generic-connector-api PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto
+target_proto_messages(connector-api-service PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto
)
-target_proto_addincls(generic-connector-api
+target_proto_addincls(connector-api-service
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -49,11 +49,11 @@ target_proto_addincls(generic-connector-api
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(generic-connector-api
+target_proto_outs(connector-api-service
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
-target_proto_plugin(generic-connector-api
+target_proto_plugin(connector-api-service
grpc_cpp
grpc_cpp
)
diff --git a/ydb/library/yql/providers/generic/connector/api/connector.proto b/ydb/library/yql/providers/generic/connector/api/service/connector.proto
index 179e4b35c1..4cf05ba27b 100644
--- a/ydb/library/yql/providers/generic/connector/api/connector.proto
+++ b/ydb/library/yql/providers/generic/connector/api/service/connector.proto
@@ -2,7 +2,7 @@ syntax = "proto3";
package NYql.Connector.API;
-import "ydb/library/yql/providers/generic/connector/api/protos/connector.proto";
+import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto";
// NOTE: Protobuf-generated code for Go will appear in yql (non-public) path
option go_package = "a.yandex-team.ru/yql/providers/connector/libgo/gen";
diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.darwin-x86_64.txt
index a5b0c41bd7..f37533ca6f 100644
--- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.darwin-x86_64.txt
@@ -19,17 +19,18 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(connector-api-protos)
-target_link_libraries(connector-api-protos PUBLIC
+add_library(api-service-protos)
+target_link_libraries(api-service-protos PUBLIC
contrib-libs-cxxsupp
yutil
+ connector-api-common
api-protos
contrib-libs-protobuf
)
-target_proto_messages(connector-api-protos PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto
+target_proto_messages(api-service-protos PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto
)
-target_proto_addincls(connector-api-protos
+target_proto_addincls(api-service-protos
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -38,7 +39,7 @@ target_proto_addincls(connector-api-protos
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(connector-api-protos
+target_proto_outs(api-service-protos
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-aarch64.txt
index f428b3ddec..f6fb469b6c 100644
--- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-aarch64.txt
@@ -19,18 +19,19 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(connector-api-protos)
-target_link_libraries(connector-api-protos PUBLIC
+add_library(api-service-protos)
+target_link_libraries(api-service-protos PUBLIC
contrib-libs-linux-headers
contrib-libs-cxxsupp
yutil
+ connector-api-common
api-protos
contrib-libs-protobuf
)
-target_proto_messages(connector-api-protos PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto
+target_proto_messages(api-service-protos PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto
)
-target_proto_addincls(connector-api-protos
+target_proto_addincls(api-service-protos
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -39,7 +40,7 @@ target_proto_addincls(connector-api-protos
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(connector-api-protos
+target_proto_outs(api-service-protos
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-x86_64.txt
index f428b3ddec..f6fb469b6c 100644
--- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-x86_64.txt
@@ -19,18 +19,19 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(connector-api-protos)
-target_link_libraries(connector-api-protos PUBLIC
+add_library(api-service-protos)
+target_link_libraries(api-service-protos PUBLIC
contrib-libs-linux-headers
contrib-libs-cxxsupp
yutil
+ connector-api-common
api-protos
contrib-libs-protobuf
)
-target_proto_messages(connector-api-protos PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto
+target_proto_messages(api-service-protos PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto
)
-target_proto_addincls(connector-api-protos
+target_proto_addincls(api-service-protos
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -39,7 +40,7 @@ target_proto_addincls(connector-api-protos
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(connector-api-protos
+target_proto_outs(api-service-protos
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
diff --git a/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.txt
new file mode 100644
index 0000000000..f8b31df0c1
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.windows-x86_64.txt
index a5b0c41bd7..f37533ca6f 100644
--- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.windows-x86_64.txt
@@ -19,17 +19,18 @@ get_built_tool_path(
cpp_styleguide
)
-add_library(connector-api-protos)
-target_link_libraries(connector-api-protos PUBLIC
+add_library(api-service-protos)
+target_link_libraries(api-service-protos PUBLIC
contrib-libs-cxxsupp
yutil
+ connector-api-common
api-protos
contrib-libs-protobuf
)
-target_proto_messages(connector-api-protos PRIVATE
- ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto
+target_proto_messages(api-service-protos PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto
)
-target_proto_addincls(connector-api-protos
+target_proto_addincls(api-service-protos
./
${CMAKE_SOURCE_DIR}/
${CMAKE_BINARY_DIR}
@@ -38,7 +39,7 @@ target_proto_addincls(connector-api-protos
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src
)
-target_proto_outs(connector-api-protos
+target_proto_outs(api-service-protos
--cpp_out=${CMAKE_BINARY_DIR}/
--cpp_styleguide_out=${CMAKE_BINARY_DIR}/
)
diff --git a/ydb/library/yql/providers/generic/connector/api/protos/connector.proto b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto
index 760569b4d9..1b66004478 100644
--- a/ydb/library/yql/providers/generic/connector/api/protos/connector.proto
+++ b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto
@@ -5,6 +5,7 @@ package NYql.Connector.API;
import "ydb/public/api/protos/ydb_value.proto";
import "ydb/public/api/protos/ydb_status_codes.proto";
import "ydb/public/api/protos/ydb_issue_message.proto";
+import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto";
// NOTE: Protobuf-generated code for Go will appear in yql (non-public) path
option go_package = "a.yandex-team.ru/yql/providers/connector/libgo/gen/protos";
@@ -398,45 +399,6 @@ message Filter {
// ---------- Utils ----------
-// Describes the kind of external data source
-enum DataSourceKind {
- DATA_SOURCE_KIND_RESERVED = 0;
- CLICKHOUSE = 1;
- POSTGRESQL = 2;
-}
-
-message Endpoint {
- string host = 1;
- uint32 port = 2;
-}
-
-message Credentials {
- // Basic
- message Basic {
- string username = 1;
- string password = 2;
- }
-
- oneof payload {
- Basic basic = 1;
- }
-}
-
-// DataSourceInstance helps to identify the instance of a data source to route request to.
-message DataSourceInstance {
- // Kind of the DataSource
- DataSourceKind kind = 1;
-
- // Network address to connect
- Endpoint endpoint = 2;
-
- // What database to connect
- string database = 3;
-
- // Information for authentication
- Credentials credentials = 4;
-}
-
// Special type to describe the result of any operation
message Error {
// High-level code
diff --git a/ydb/library/yql/providers/generic/connector/api/protos/ya.make b/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make
index b6f247a92e..f9fe5ea3ca 100644
--- a/ydb/library/yql/providers/generic/connector/api/protos/ya.make
+++ b/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make
@@ -1,6 +1,7 @@
PROTO_LIBRARY()
PEERDIR(
+ ydb/library/yql/providers/generic/connector/api/common
ydb/public/api/protos
)
diff --git a/ydb/library/yql/providers/generic/connector/api/service/ya.make b/ydb/library/yql/providers/generic/connector/api/service/ya.make
new file mode 100644
index 0000000000..8bd333be0e
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/api/service/ya.make
@@ -0,0 +1,16 @@
+PROTO_LIBRARY()
+
+GRPC()
+
+SRCS(
+ connector.proto
+)
+
+# Because Go is excluded in YDB protofiles
+EXCLUDE_TAGS(GO_PROTO)
+
+PEERDIR(
+ ydb/library/yql/providers/generic/connector/api/service/protos
+)
+
+END()
diff --git a/ydb/library/yql/providers/generic/connector/api/ya.make b/ydb/library/yql/providers/generic/connector/api/ya.make
index b255d99f9a..f828c33ddc 100644
--- a/ydb/library/yql/providers/generic/connector/api/ya.make
+++ b/ydb/library/yql/providers/generic/connector/api/ya.make
@@ -1,16 +1,4 @@
-PROTO_LIBRARY()
-
-GRPC()
-
-SRCS(
- connector.proto
+RECURSE(
+ common # Keep this library pure of dependencies on YDB protofiles
+ service # Use YDB protofiles only here
)
-
-# Because Go is excluded in YDB protofiles
-EXCLUDE_TAGS(GO_PROTO)
-
-PEERDIR(
- ydb/library/yql/providers/generic/connector/api/protos
-)
-
-END()
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt
index 4578636bd8..9e314d464c 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt
@@ -16,9 +16,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC
contrib-libs-grpc
formats-arrow-serializer
library-yql-ast
+ connector-api-common
+ connector-api-service
yql-public-issue
library-yql-utils
- generic-connector-api
)
target_sources(generic-connector-libcpp PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt
index dc40e3f62e..42a58b0982 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt
@@ -17,9 +17,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC
contrib-libs-grpc
formats-arrow-serializer
library-yql-ast
+ connector-api-common
+ connector-api-service
yql-public-issue
library-yql-utils
- generic-connector-api
)
target_sources(generic-connector-libcpp PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt
index dc40e3f62e..42a58b0982 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt
@@ -17,9 +17,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC
contrib-libs-grpc
formats-arrow-serializer
library-yql-ast
+ connector-api-common
+ connector-api-service
yql-public-issue
library-yql-utils
- generic-connector-api
)
target_sources(generic-connector-libcpp PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt
index 4578636bd8..9e314d464c 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt
@@ -16,9 +16,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC
contrib-libs-grpc
formats-arrow-serializer
library-yql-ast
+ connector-api-common
+ connector-api-service
yql-public-issue
library-yql-utils
- generic-connector-api
)
target_sources(generic-connector-libcpp PRIVATE
${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/client.h b/ydb/library/yql/providers/generic/connector/libcpp/client.h
index a955288215..e36fef9304 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/client.h
+++ b/ydb/library/yql/providers/generic/connector/libcpp/client.h
@@ -2,8 +2,8 @@
#include <arrow/api.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
-#include <ydb/library/yql/providers/generic/connector/api/connector.grpc.pb.h>
-#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h>
+#include <ydb/library/yql/providers/generic/connector/api/service/connector.grpc.pb.h>
+#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h>
namespace NYql::Connector {
struct DescribeTableResult {
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp b/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
index 17466f04f9..d14a1d8f20 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
+++ b/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp
@@ -5,7 +5,7 @@
#include <grpcpp/client_context.h>
#include <grpcpp/create_channel.h>
#include <ydb/library/yql/utils/log/log.h>
-#include <ydb/library/yql/providers/generic/connector/api/connector.grpc.pb.h>
+#include <ydb/library/yql/providers/generic/connector/api/service/connector.grpc.pb.h>
#define REQUEST_START() ({ YQL_LOG(INFO) << __func__ << ": request handling started"; })
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/error.h b/ydb/library/yql/providers/generic/connector/libcpp/error.h
index 9e1f13f323..0e2d32f866 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/error.h
+++ b/ydb/library/yql/providers/generic/connector/libcpp/error.h
@@ -2,7 +2,7 @@
#include <grpcpp/support/status.h>
#include <ydb/library/yql/ast/yql_expr.h>
-#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h>
+#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h>
namespace NYql::Connector {
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/utils.h b/ydb/library/yql/providers/generic/connector/libcpp/utils.h
index 5017a339e0..2f628e23f8 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/utils.h
+++ b/ydb/library/yql/providers/generic/connector/libcpp/utils.h
@@ -1,7 +1,7 @@
#pragma once
#include <arrow/api.h>
-#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h>
+#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h>
namespace NYql::Connector {
diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ya.make b/ydb/library/yql/providers/generic/connector/libcpp/ya.make
index c94ce96f17..55bbc64577 100644
--- a/ydb/library/yql/providers/generic/connector/libcpp/ya.make
+++ b/ydb/library/yql/providers/generic/connector/libcpp/ya.make
@@ -12,9 +12,10 @@ PEERDIR(
contrib/libs/grpc
ydb/core/formats/arrow/serializer
ydb/library/yql/ast
+ ydb/library/yql/providers/generic/connector/api/common
+ ydb/library/yql/providers/generic/connector/api/service
ydb/library/yql/public/issue
ydb/library/yql/utils
- ydb/library/yql/providers/generic/connector/api
)
END()
diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt
index 3eb5e86f97..9d2318ef89 100644
--- a/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt
@@ -35,7 +35,8 @@ add_library(providers-generic-proto)
target_link_libraries(providers-generic-proto PUBLIC
contrib-libs-cxxsupp
yutil
- connector-api-protos
+ api-service-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-generic-proto PRIVATE
diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt
index d8e7b08f05..4e41576ab2 100644
--- a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt
@@ -36,7 +36,8 @@ target_link_libraries(providers-generic-proto PUBLIC
contrib-libs-linux-headers
contrib-libs-cxxsupp
yutil
- connector-api-protos
+ api-service-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-generic-proto PRIVATE
diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt
index d8e7b08f05..4e41576ab2 100644
--- a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt
@@ -36,7 +36,8 @@ target_link_libraries(providers-generic-proto PUBLIC
contrib-libs-linux-headers
contrib-libs-cxxsupp
yutil
- connector-api-protos
+ api-service-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-generic-proto PRIVATE
diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt
index 3eb5e86f97..9d2318ef89 100644
--- a/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt
@@ -35,7 +35,8 @@ add_library(providers-generic-proto)
target_link_libraries(providers-generic-proto PUBLIC
contrib-libs-cxxsupp
yutil
- connector-api-protos
+ api-service-protos
+ connector-api-common
contrib-libs-protobuf
)
target_proto_messages(providers-generic-proto PRIVATE
diff --git a/ydb/library/yql/providers/generic/proto/source.proto b/ydb/library/yql/providers/generic/proto/source.proto
index 284af4530f..2f616be706 100644
--- a/ydb/library/yql/providers/generic/proto/source.proto
+++ b/ydb/library/yql/providers/generic/proto/source.proto
@@ -4,7 +4,8 @@ option cc_enable_arenas = true;
package NYql.Generic;
-import "ydb/library/yql/providers/generic/connector/api/protos/connector.proto";
+import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto";
+import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto";
message TSource {
// Token to access database
diff --git a/ydb/library/yql/providers/generic/proto/ya.make b/ydb/library/yql/providers/generic/proto/ya.make
index e757b582b8..350d192c15 100644
--- a/ydb/library/yql/providers/generic/proto/ya.make
+++ b/ydb/library/yql/providers/generic/proto/ya.make
@@ -3,7 +3,8 @@ PROTO_LIBRARY()
ONLY_TAGS(CPP_PROTO)
PEERDIR(
- ydb/library/yql/providers/generic/connector/api/protos
+ ydb/library/yql/providers/generic/connector/api/service/protos
+ ydb/library/yql/providers/generic/connector/api/common
)
SRCS(
diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h
index 36a555f252..3d6f36d722 100644
--- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h
+++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h
@@ -4,7 +4,7 @@
#include <ydb/library/yql/providers/common/config/yql_setting.h>
#include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h>
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
-#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h>
+#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h>
#include <ydb/library/yql/utils/log/log.h>
namespace NYql {