diff options
author | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-06-19 16:09:10 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-06-19 16:09:10 +0300 |
commit | c7fa117ab3122dd361bcc50dd975c12bb6714bbb (patch) | |
tree | 3eda0ae81f35ad70686dbf99b6f9fcc2cd1ce6f7 | |
parent | 128fa513a8669e87b8e2322efa61c7b82a8c085d (diff) | |
download | ydb-c7fa117ab3122dd361bcc50dd975c12bb6714bbb.tar.gz |
Prepare Connector for deployment in Cloud
1. В сервер добавлен режим работы с TLS и протестирован с помощью клиента на самоподписных сертификатах.
2. Конфигурации клиента и сервера теперь описаны в Protobuf, при этом они переиспользуют некоторые типы из API Коннектора.
3. В связи с п. 2 API отрефакторено и разделено по двум папкам: `ydb/library/yql/providers/generic/connector/api/common` и `ydb/library/yql/providers/generic/connector/api/service`
4. Добавлена большая часть скриптов для сборки через `ya package` и разворачивания сервиса (по аналогии с `yq-watchdog`).
51 files changed, 650 insertions, 148 deletions
diff --git a/library/go/core/buildinfo/test/main.go b/library/go/core/buildinfo/test/main.go new file mode 100644 index 0000000000..a17ac12749 --- /dev/null +++ b/library/go/core/buildinfo/test/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + + "a.yandex-team.ru/library/go/core/buildinfo" +) + +func main() { + if buildinfo.Info.ProgramVersion != "" { + fmt.Print(buildinfo.Info.ProgramVersion) + } else { + fmt.Printf("ProgramVersion is not available\n") + } +} diff --git a/library/go/core/buildinfo/test/ya.make b/library/go/core/buildinfo/test/ya.make new file mode 100644 index 0000000000..6a0765382d --- /dev/null +++ b/library/go/core/buildinfo/test/ya.make @@ -0,0 +1,5 @@ +GO_PROGRAM() + +SRCS(main.go) + +END() diff --git a/tools/go_test_miner/main.go b/tools/go_test_miner/main.go new file mode 100644 index 0000000000..43b729572e --- /dev/null +++ b/tools/go_test_miner/main.go @@ -0,0 +1,168 @@ +package main + +import ( + "flag" + "fmt" + "go/importer" + "go/token" + "go/types" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +const ( + usageTemplate = "Usage: %s [-benchmarks] [-examples] [-tests] import-path\n" +) + +func findObjectByName(pkg *types.Package, re *regexp.Regexp, name string) types.Object { + if pkg != nil && re != nil && len(name) > 0 { + if obj := pkg.Scope().Lookup(name); obj != nil { + if re.MatchString(obj.Type().String()) { + return obj + } + } + } + return nil +} + +func isTestName(name, prefix string) bool { + ok := false + if strings.HasPrefix(name, prefix) { + if len(name) == len(prefix) { + ok = true + } else { + rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) + ok = !unicode.IsLower(rune) + } + } + return ok +} + +func main() { + testsPtr := flag.Bool("tests", false, "report tests") + benchmarksPtr := flag.Bool("benchmarks", false, "report benchmarks") + examplesPtr := flag.Bool("examples", false, "report examples") + + flag.Usage = func() { + _, _ = fmt.Fprintf(flag.CommandLine.Output(), usageTemplate, filepath.Base(os.Args[0])) + flag.PrintDefaults() + } + + flag.Parse() + + // Check if the number of positional parameters matches + args := flag.Args() + argsCount := len(args) + if argsCount != 1 { + exitCode := 0 + if argsCount > 1 { + fmt.Println("Error: invalid number of parameters...") + exitCode = 1 + } + flag.Usage() + os.Exit(exitCode) + } + + importPath := args[0] + + var fset token.FileSet + imp := importer.ForCompiler(&fset, runtime.Compiler, nil) + pkg, err := imp.Import(importPath) + if err != nil { + fmt.Printf("Error: %v\n", err) + os.Exit(1) + } + + if !*testsPtr && !*benchmarksPtr && !*examplesPtr { + // Nothing to do, just exit normally + os.Exit(0) + } + + // // First approach: just dump the package scope as a string + // // package "junk/snermolaev/libmath" scope 0xc0000df540 { + // // . func junk/snermolaev/libmath.Abs(a int) int + // // . func junk/snermolaev/libmath.AbsReport(s string) + // // . func junk/snermolaev/libmath.Sum(a int, b int) int + // // . func junk/snermolaev/libmath.TestAbs(t *testing.T) + // // . func junk/snermolaev/libmath.TestSum(t *testing.T) + // // . func junk/snermolaev/libmath.init() + // // } + // // and then collect all functions that match test function signature + // pkgPath := pkg.Path() + // scopeContent := strings.Split(pkg.Scope().String(), "\n") + // re := regexp.MustCompile("^\\.\\s*func\\s*" + pkgPath + "\\.(Test\\w*)\\(\\s*\\w*\\s*\\*\\s*testing\\.T\\s*\\)$") + // for _, name := range scopeContent { + // match := re.FindAllStringSubmatch(name, -1) + // if len(match) > 0 { + // fmt.Println(match[0][1]) + // } + // } + + // Second approach: look through all names defined in the pkg scope + // and collect those functions that match test function signature + // Unfortunately I failed to employ reflection mechinary for signature + // comparison for unknown reasons (this needs additional investigation + // I am going to use regexp as workaround for a while) + // testFunc := func (*testing.T) {} + // for ... + // ... + // if reflect.DeepEqual(obj.Type(), reflect.TypeOf(testFunc)) { + // // this condition doesn't work + // } + reBenchmark := regexp.MustCompile(`^func\(\w*\s*\*testing\.B\)$`) + reExample := regexp.MustCompile(`^func\(\s*\)$`) + reTest := regexp.MustCompile(`^func\(\w*\s*\*testing\.T\)$`) + reTestMain := regexp.MustCompile(`^func\(\w*\s*\*testing\.M\)$`) + + var re *regexp.Regexp + names := pkg.Scope().Names() + + var testFns []types.Object + + for _, name := range names { + if name == "TestMain" && findObjectByName(pkg, reTestMain, name) != nil { + fmt.Println("#TestMain") + continue + } + + switch { + case *benchmarksPtr && isTestName(name, "Benchmark"): + re = reBenchmark + case *examplesPtr && isTestName(name, "Example"): + re = reExample + case *testsPtr && isTestName(name, "Test"): + re = reTest + default: + continue + } + + if obj := findObjectByName(pkg, re, name); obj != nil { + testFns = append(testFns, obj) + } + } + + sort.Slice(testFns, func(i, j int) bool { + iPos := testFns[i].Pos() + jPos := testFns[j].Pos() + + if !iPos.IsValid() || !jPos.IsValid() { + return iPos < jPos + } + + iPosition := fset.PositionFor(iPos, true) + jPosition := fset.PositionFor(jPos, true) + + return iPosition.Filename < jPosition.Filename || + (iPosition.Filename == jPosition.Filename && iPosition.Line < jPosition.Line) + }) + + for _, testFn := range testFns { + fmt.Println(testFn.Name()) + } +} diff --git a/tools/go_test_miner/ya.make b/tools/go_test_miner/ya.make new file mode 100644 index 0000000000..2efea0ec7b --- /dev/null +++ b/tools/go_test_miner/ya.make @@ -0,0 +1,16 @@ +GO_PROGRAM() + +SRCS( + main.go +) + +GO_TEST_SRCS( + main_test.go +) + +END() + +RECURSE_FOR_TESTS( + gotest +) + diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt index 0841a72dae..962067cd49 100644 --- a/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/common/proto/CMakeLists.darwin-x86_64.txt @@ -36,7 +36,7 @@ target_link_libraries(providers-common-proto PUBLIC contrib-libs-cxxsupp yutil library-yql-protos - connector-api-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-common-proto PRIVATE diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt index 8dade7c78c..9925b68fbf 100644 --- a/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/common/proto/CMakeLists.linux-aarch64.txt @@ -37,7 +37,7 @@ target_link_libraries(providers-common-proto PUBLIC contrib-libs-cxxsupp yutil library-yql-protos - connector-api-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-common-proto PRIVATE diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt index 8dade7c78c..9925b68fbf 100644 --- a/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/common/proto/CMakeLists.linux-x86_64.txt @@ -37,7 +37,7 @@ target_link_libraries(providers-common-proto PUBLIC contrib-libs-cxxsupp yutil library-yql-protos - connector-api-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-common-proto PRIVATE diff --git a/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt index 0841a72dae..962067cd49 100644 --- a/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/common/proto/CMakeLists.windows-x86_64.txt @@ -36,7 +36,7 @@ target_link_libraries(providers-common-proto PUBLIC contrib-libs-cxxsupp yutil library-yql-protos - connector-api-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-common-proto PRIVATE diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index 24c82e2dc7..1a96f12b17 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -2,7 +2,8 @@ package NYql; option java_package = "ru.yandex.yql.proto"; import "ydb/library/yql/protos/clickhouse.proto"; -import "ydb/library/yql/providers/generic/connector/api/protos/connector.proto"; +import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto"; +import "ydb/library/yql/providers/generic/connector/api/common/endpoint.proto"; /////////////////////////////// common /////////////////////////////// @@ -556,6 +557,7 @@ message TGenericClusterConfig { } message TGenericGatewayConfig { + // TODO: replace with map<DataSourceKind, Endpoint> required string Endpoint = 1; reserved 2; repeated TGenericClusterConfig ClusterMapping = 3; diff --git a/ydb/library/yql/providers/common/proto/ya.make b/ydb/library/yql/providers/common/proto/ya.make index 3c3ffac95d..fded415f56 100644 --- a/ydb/library/yql/providers/common/proto/ya.make +++ b/ydb/library/yql/providers/common/proto/ya.make @@ -7,7 +7,7 @@ SRCS( PEERDIR( ydb/library/yql/protos - ydb/library/yql/providers/generic/connector/api/protos + ydb/library/yql/providers/generic/connector/api/common ) IF (NOT PY_PROTOS_FOR) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index 8e49d47c74..70345916a6 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -12,7 +12,7 @@ #include <ydb/library/yql/public/udf/arrow/util.h> #include <ydb/library/yql/utils/log/log.h> #include <ydb/library/yql/utils/yql_panic.h> -#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h> +#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h> #include <ydb/library/yql/providers/generic/connector/libcpp/error.h> namespace NYql::NDq { diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt index f8b31df0c1..459e4be130 100644 --- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt +++ b/ydb/library/yql/providers/generic/connector/api/CMakeLists.txt @@ -6,12 +6,5 @@ # original buildsystem will not be accepted. -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) - include(CMakeLists.windows-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) - include(CMakeLists.linux-x86_64.txt) -endif() +add_subdirectory(common) +add_subdirectory(service) diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..1af4185e07 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,56 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(connector-api-common) +target_link_libraries(connector-api-common PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(connector-api-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto +) +target_proto_addincls(connector-api-common + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(connector-api-common + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..ca49587737 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-aarch64.txt @@ -0,0 +1,57 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(connector-api-common) +target_link_libraries(connector-api-common PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(connector-api-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto +) +target_proto_addincls(connector-api-common + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(connector-api-common + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..ca49587737 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.linux-x86_64.txt @@ -0,0 +1,57 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(connector-api-common) +target_link_libraries(connector-api-common PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(connector-api-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto +) +target_proto_addincls(connector-api-common + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(connector-api-common + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.txt index f8b31df0c1..f8b31df0c1 100644 --- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.txt +++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.txt diff --git a/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..1af4185e07 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/common/CMakeLists.windows-x86_64.txt @@ -0,0 +1,56 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) +get_built_tool_path( + TOOL_protoc_bin + TOOL_protoc_dependency + contrib/tools/protoc/bin + protoc +) +get_built_tool_path( + TOOL_cpp_styleguide_bin + TOOL_cpp_styleguide_dependency + contrib/tools/protoc/plugins/cpp_styleguide + cpp_styleguide +) + +add_library(connector-api-common) +target_link_libraries(connector-api-common PUBLIC + contrib-libs-cxxsupp + yutil + contrib-libs-protobuf +) +target_proto_messages(connector-api-common PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/data_source.proto + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto +) +target_proto_addincls(connector-api-common + ./ + ${CMAKE_SOURCE_DIR}/ + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src +) +target_proto_outs(connector-api-common + --cpp_out=${CMAKE_BINARY_DIR}/ + --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ +) diff --git a/ydb/library/yql/providers/generic/connector/api/common/data_source.proto b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto new file mode 100644 index 0000000000..5c64d09711 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/common/data_source.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package NYql.Connector.API; + +import "ydb/library/yql/providers/generic/connector/api/common/endpoint.proto"; + +option go_package = "a.yandex-team.ru/ydb/library/yql/providers/generic/connector/api/common"; + +// Credentials represents various ways of user authentication in the data source instance +message Credentials { + // Basic + message Basic { + string username = 1; + string password = 2; + } + + oneof payload { + Basic basic = 1; + } +} + +// DataSourceKind enumerates the external data sources +// supported in the federated query system +enum DataSourceKind { + DATA_SOURCE_KIND_RESERVED = 0; + CLICKHOUSE = 1; + POSTGRESQL = 2; +} + +// DataSourceInstance helps to identify the instance of a data source to route request to. +message DataSourceInstance { + DataSourceKind kind = 1; + Endpoint endpoint = 2; + + // database name as it is + string database = 3; + + Credentials credentials = 4; +} diff --git a/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto b/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto new file mode 100644 index 0000000000..5b8ce1f2bd --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/common/endpoint.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package NYql.Connector.API; + +option go_package = "a.yandex-team.ru/ydb/library/yql/providers/generic/connector/api/common"; + +// Endpoint represents the network address of a data source instance +message Endpoint { + string host = 1; + uint32 port = 2; +}
\ No newline at end of file diff --git a/ydb/library/yql/providers/generic/connector/api/common/ya.make b/ydb/library/yql/providers/generic/connector/api/common/ya.make new file mode 100644 index 0000000000..0dd3d856cb --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/common/ya.make @@ -0,0 +1,11 @@ +# WARNING: never add dependency to YDB protofiles here, +# because it would break Go code generation. + +PROTO_LIBRARY() + +SRCS( + data_source.proto + endpoint.proto +) + +END() diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.darwin-x86_64.txt index fc57461fef..75d63720a6 100644 --- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.darwin-x86_64.txt @@ -26,21 +26,21 @@ get_built_tool_path( cpp_styleguide ) -add_library(generic-connector-api) -set_property(TARGET generic-connector-api PROPERTY +add_library(connector-api-service) +set_property(TARGET connector-api-service PROPERTY PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h ) -target_link_libraries(generic-connector-api PUBLIC +target_link_libraries(connector-api-service PUBLIC contrib-libs-cxxsupp yutil contrib-libs-grpc - connector-api-protos + api-service-protos contrib-libs-protobuf ) -target_proto_messages(generic-connector-api PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto +target_proto_messages(connector-api-service PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto ) -target_proto_addincls(generic-connector-api +target_proto_addincls(connector-api-service ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -49,11 +49,11 @@ target_proto_addincls(generic-connector-api ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(generic-connector-api +target_proto_outs(connector-api-service --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) -target_proto_plugin(generic-connector-api +target_proto_plugin(connector-api-service grpc_cpp grpc_cpp ) diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-aarch64.txt index 1e9bb96043..b41c8ae098 100644 --- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-aarch64.txt @@ -26,22 +26,22 @@ get_built_tool_path( cpp_styleguide ) -add_library(generic-connector-api) -set_property(TARGET generic-connector-api PROPERTY +add_library(connector-api-service) +set_property(TARGET connector-api-service PROPERTY PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h ) -target_link_libraries(generic-connector-api PUBLIC +target_link_libraries(connector-api-service PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil contrib-libs-grpc - connector-api-protos + api-service-protos contrib-libs-protobuf ) -target_proto_messages(generic-connector-api PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto +target_proto_messages(connector-api-service PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto ) -target_proto_addincls(generic-connector-api +target_proto_addincls(connector-api-service ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -50,11 +50,11 @@ target_proto_addincls(generic-connector-api ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(generic-connector-api +target_proto_outs(connector-api-service --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) -target_proto_plugin(generic-connector-api +target_proto_plugin(connector-api-service grpc_cpp grpc_cpp ) diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-x86_64.txt index 1e9bb96043..b41c8ae098 100644 --- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.linux-x86_64.txt @@ -26,22 +26,22 @@ get_built_tool_path( cpp_styleguide ) -add_library(generic-connector-api) -set_property(TARGET generic-connector-api PROPERTY +add_library(connector-api-service) +set_property(TARGET connector-api-service PROPERTY PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h ) -target_link_libraries(generic-connector-api PUBLIC +target_link_libraries(connector-api-service PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil contrib-libs-grpc - connector-api-protos + api-service-protos contrib-libs-protobuf ) -target_proto_messages(generic-connector-api PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto +target_proto_messages(connector-api-service PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto ) -target_proto_addincls(generic-connector-api +target_proto_addincls(connector-api-service ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -50,11 +50,11 @@ target_proto_addincls(generic-connector-api ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(generic-connector-api +target_proto_outs(connector-api-service --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) -target_proto_plugin(generic-connector-api +target_proto_plugin(connector-api-service grpc_cpp grpc_cpp ) diff --git a/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/generic/connector/api/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.windows-x86_64.txt index fc57461fef..75d63720a6 100644 --- a/ydb/library/yql/providers/generic/connector/api/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/CMakeLists.windows-x86_64.txt @@ -26,21 +26,21 @@ get_built_tool_path( cpp_styleguide ) -add_library(generic-connector-api) -set_property(TARGET generic-connector-api PROPERTY +add_library(connector-api-service) +set_property(TARGET connector-api-service PROPERTY PROTOC_EXTRA_OUTS .grpc.pb.cc .grpc.pb.h ) -target_link_libraries(generic-connector-api PUBLIC +target_link_libraries(connector-api-service PUBLIC contrib-libs-cxxsupp yutil contrib-libs-grpc - connector-api-protos + api-service-protos contrib-libs-protobuf ) -target_proto_messages(generic-connector-api PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/connector.proto +target_proto_messages(connector-api-service PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/connector.proto ) -target_proto_addincls(generic-connector-api +target_proto_addincls(connector-api-service ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -49,11 +49,11 @@ target_proto_addincls(generic-connector-api ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(generic-connector-api +target_proto_outs(connector-api-service --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) -target_proto_plugin(generic-connector-api +target_proto_plugin(connector-api-service grpc_cpp grpc_cpp ) diff --git a/ydb/library/yql/providers/generic/connector/api/connector.proto b/ydb/library/yql/providers/generic/connector/api/service/connector.proto index 179e4b35c1..4cf05ba27b 100644 --- a/ydb/library/yql/providers/generic/connector/api/connector.proto +++ b/ydb/library/yql/providers/generic/connector/api/service/connector.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package NYql.Connector.API; -import "ydb/library/yql/providers/generic/connector/api/protos/connector.proto"; +import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto"; // NOTE: Protobuf-generated code for Go will appear in yql (non-public) path option go_package = "a.yandex-team.ru/yql/providers/connector/libgo/gen"; diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.darwin-x86_64.txt index a5b0c41bd7..f37533ca6f 100644 --- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.darwin-x86_64.txt @@ -19,17 +19,18 @@ get_built_tool_path( cpp_styleguide ) -add_library(connector-api-protos) -target_link_libraries(connector-api-protos PUBLIC +add_library(api-service-protos) +target_link_libraries(api-service-protos PUBLIC contrib-libs-cxxsupp yutil + connector-api-common api-protos contrib-libs-protobuf ) -target_proto_messages(connector-api-protos PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto +target_proto_messages(api-service-protos PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto ) -target_proto_addincls(connector-api-protos +target_proto_addincls(api-service-protos ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -38,7 +39,7 @@ target_proto_addincls(connector-api-protos ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(connector-api-protos +target_proto_outs(api-service-protos --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-aarch64.txt index f428b3ddec..f6fb469b6c 100644 --- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-aarch64.txt @@ -19,18 +19,19 @@ get_built_tool_path( cpp_styleguide ) -add_library(connector-api-protos) -target_link_libraries(connector-api-protos PUBLIC +add_library(api-service-protos) +target_link_libraries(api-service-protos PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + connector-api-common api-protos contrib-libs-protobuf ) -target_proto_messages(connector-api-protos PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto +target_proto_messages(api-service-protos PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto ) -target_proto_addincls(connector-api-protos +target_proto_addincls(api-service-protos ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -39,7 +40,7 @@ target_proto_addincls(connector-api-protos ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(connector-api-protos +target_proto_outs(api-service-protos --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-x86_64.txt index f428b3ddec..f6fb469b6c 100644 --- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.linux-x86_64.txt @@ -19,18 +19,19 @@ get_built_tool_path( cpp_styleguide ) -add_library(connector-api-protos) -target_link_libraries(connector-api-protos PUBLIC +add_library(api-service-protos) +target_link_libraries(api-service-protos PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil + connector-api-common api-protos contrib-libs-protobuf ) -target_proto_messages(connector-api-protos PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto +target_proto_messages(api-service-protos PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto ) -target_proto_addincls(connector-api-protos +target_proto_addincls(api-service-protos ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -39,7 +40,7 @@ target_proto_addincls(connector-api-protos ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(connector-api-protos +target_proto_outs(api-service-protos --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) diff --git a/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.windows-x86_64.txt index a5b0c41bd7..f37533ca6f 100644 --- a/ydb/library/yql/providers/generic/connector/api/protos/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/CMakeLists.windows-x86_64.txt @@ -19,17 +19,18 @@ get_built_tool_path( cpp_styleguide ) -add_library(connector-api-protos) -target_link_libraries(connector-api-protos PUBLIC +add_library(api-service-protos) +target_link_libraries(api-service-protos PUBLIC contrib-libs-cxxsupp yutil + connector-api-common api-protos contrib-libs-protobuf ) -target_proto_messages(connector-api-protos PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/protos/connector.proto +target_proto_messages(api-service-protos PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto ) -target_proto_addincls(connector-api-protos +target_proto_addincls(api-service-protos ./ ${CMAKE_SOURCE_DIR}/ ${CMAKE_BINARY_DIR} @@ -38,7 +39,7 @@ target_proto_addincls(connector-api-protos ${CMAKE_BINARY_DIR} ${CMAKE_SOURCE_DIR}/contrib/libs/protobuf/src ) -target_proto_outs(connector-api-protos +target_proto_outs(api-service-protos --cpp_out=${CMAKE_BINARY_DIR}/ --cpp_styleguide_out=${CMAKE_BINARY_DIR}/ ) diff --git a/ydb/library/yql/providers/generic/connector/api/protos/connector.proto b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto index 760569b4d9..1b66004478 100644 --- a/ydb/library/yql/providers/generic/connector/api/protos/connector.proto +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto @@ -5,6 +5,7 @@ package NYql.Connector.API; import "ydb/public/api/protos/ydb_value.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; +import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto"; // NOTE: Protobuf-generated code for Go will appear in yql (non-public) path option go_package = "a.yandex-team.ru/yql/providers/connector/libgo/gen/protos"; @@ -398,45 +399,6 @@ message Filter { // ---------- Utils ---------- -// Describes the kind of external data source -enum DataSourceKind { - DATA_SOURCE_KIND_RESERVED = 0; - CLICKHOUSE = 1; - POSTGRESQL = 2; -} - -message Endpoint { - string host = 1; - uint32 port = 2; -} - -message Credentials { - // Basic - message Basic { - string username = 1; - string password = 2; - } - - oneof payload { - Basic basic = 1; - } -} - -// DataSourceInstance helps to identify the instance of a data source to route request to. -message DataSourceInstance { - // Kind of the DataSource - DataSourceKind kind = 1; - - // Network address to connect - Endpoint endpoint = 2; - - // What database to connect - string database = 3; - - // Information for authentication - Credentials credentials = 4; -} - // Special type to describe the result of any operation message Error { // High-level code diff --git a/ydb/library/yql/providers/generic/connector/api/protos/ya.make b/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make index b6f247a92e..f9fe5ea3ca 100644 --- a/ydb/library/yql/providers/generic/connector/api/protos/ya.make +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/ya.make @@ -1,6 +1,7 @@ PROTO_LIBRARY() PEERDIR( + ydb/library/yql/providers/generic/connector/api/common ydb/public/api/protos ) diff --git a/ydb/library/yql/providers/generic/connector/api/service/ya.make b/ydb/library/yql/providers/generic/connector/api/service/ya.make new file mode 100644 index 0000000000..8bd333be0e --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/api/service/ya.make @@ -0,0 +1,16 @@ +PROTO_LIBRARY() + +GRPC() + +SRCS( + connector.proto +) + +# Because Go is excluded in YDB protofiles +EXCLUDE_TAGS(GO_PROTO) + +PEERDIR( + ydb/library/yql/providers/generic/connector/api/service/protos +) + +END() diff --git a/ydb/library/yql/providers/generic/connector/api/ya.make b/ydb/library/yql/providers/generic/connector/api/ya.make index b255d99f9a..f828c33ddc 100644 --- a/ydb/library/yql/providers/generic/connector/api/ya.make +++ b/ydb/library/yql/providers/generic/connector/api/ya.make @@ -1,16 +1,4 @@ -PROTO_LIBRARY() - -GRPC() - -SRCS( - connector.proto +RECURSE( + common # Keep this library pure of dependencies on YDB protofiles + service # Use YDB protofiles only here ) - -# Because Go is excluded in YDB protofiles -EXCLUDE_TAGS(GO_PROTO) - -PEERDIR( - ydb/library/yql/providers/generic/connector/api/protos -) - -END() diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt index 4578636bd8..9e314d464c 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.darwin-x86_64.txt @@ -16,9 +16,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC contrib-libs-grpc formats-arrow-serializer library-yql-ast + connector-api-common + connector-api-service yql-public-issue library-yql-utils - generic-connector-api ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt index dc40e3f62e..42a58b0982 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-aarch64.txt @@ -17,9 +17,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC contrib-libs-grpc formats-arrow-serializer library-yql-ast + connector-api-common + connector-api-service yql-public-issue library-yql-utils - generic-connector-api ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt index dc40e3f62e..42a58b0982 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.linux-x86_64.txt @@ -17,9 +17,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC contrib-libs-grpc formats-arrow-serializer library-yql-ast + connector-api-common + connector-api-service yql-public-issue library-yql-utils - generic-connector-api ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp diff --git a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt index 4578636bd8..9e314d464c 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/generic/connector/libcpp/CMakeLists.windows-x86_64.txt @@ -16,9 +16,10 @@ target_link_libraries(generic-connector-libcpp PUBLIC contrib-libs-grpc formats-arrow-serializer library-yql-ast + connector-api-common + connector-api-service yql-public-issue library-yql-utils - generic-connector-api ) target_sources(generic-connector-libcpp PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp diff --git a/ydb/library/yql/providers/generic/connector/libcpp/client.h b/ydb/library/yql/providers/generic/connector/libcpp/client.h index a955288215..e36fef9304 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/client.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/client.h @@ -2,8 +2,8 @@ #include <arrow/api.h> #include <ydb/library/yql/public/issue/yql_issue.h> -#include <ydb/library/yql/providers/generic/connector/api/connector.grpc.pb.h> -#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h> +#include <ydb/library/yql/providers/generic/connector/api/service/connector.grpc.pb.h> +#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h> namespace NYql::Connector { struct DescribeTableResult { diff --git a/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp b/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp index 17466f04f9..d14a1d8f20 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp +++ b/ydb/library/yql/providers/generic/connector/libcpp/client_grpc.cpp @@ -5,7 +5,7 @@ #include <grpcpp/client_context.h> #include <grpcpp/create_channel.h> #include <ydb/library/yql/utils/log/log.h> -#include <ydb/library/yql/providers/generic/connector/api/connector.grpc.pb.h> +#include <ydb/library/yql/providers/generic/connector/api/service/connector.grpc.pb.h> #define REQUEST_START() ({ YQL_LOG(INFO) << __func__ << ": request handling started"; }) diff --git a/ydb/library/yql/providers/generic/connector/libcpp/error.h b/ydb/library/yql/providers/generic/connector/libcpp/error.h index 9e1f13f323..0e2d32f866 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/error.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/error.h @@ -2,7 +2,7 @@ #include <grpcpp/support/status.h> #include <ydb/library/yql/ast/yql_expr.h> -#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h> +#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h> namespace NYql::Connector { diff --git a/ydb/library/yql/providers/generic/connector/libcpp/utils.h b/ydb/library/yql/providers/generic/connector/libcpp/utils.h index 5017a339e0..2f628e23f8 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/utils.h +++ b/ydb/library/yql/providers/generic/connector/libcpp/utils.h @@ -1,7 +1,7 @@ #pragma once #include <arrow/api.h> -#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h> +#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h> namespace NYql::Connector { diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ya.make b/ydb/library/yql/providers/generic/connector/libcpp/ya.make index c94ce96f17..55bbc64577 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ya.make +++ b/ydb/library/yql/providers/generic/connector/libcpp/ya.make @@ -12,9 +12,10 @@ PEERDIR( contrib/libs/grpc ydb/core/formats/arrow/serializer ydb/library/yql/ast + ydb/library/yql/providers/generic/connector/api/common + ydb/library/yql/providers/generic/connector/api/service ydb/library/yql/public/issue ydb/library/yql/utils - ydb/library/yql/providers/generic/connector/api ) END() diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt index 3eb5e86f97..9d2318ef89 100644 --- a/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/providers/generic/proto/CMakeLists.darwin-x86_64.txt @@ -35,7 +35,8 @@ add_library(providers-generic-proto) target_link_libraries(providers-generic-proto PUBLIC contrib-libs-cxxsupp yutil - connector-api-protos + api-service-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-generic-proto PRIVATE diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt index d8e7b08f05..4e41576ab2 100644 --- a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-aarch64.txt @@ -36,7 +36,8 @@ target_link_libraries(providers-generic-proto PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - connector-api-protos + api-service-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-generic-proto PRIVATE diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt index d8e7b08f05..4e41576ab2 100644 --- a/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/providers/generic/proto/CMakeLists.linux-x86_64.txt @@ -36,7 +36,8 @@ target_link_libraries(providers-generic-proto PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil - connector-api-protos + api-service-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-generic-proto PRIVATE diff --git a/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt index 3eb5e86f97..9d2318ef89 100644 --- a/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/providers/generic/proto/CMakeLists.windows-x86_64.txt @@ -35,7 +35,8 @@ add_library(providers-generic-proto) target_link_libraries(providers-generic-proto PUBLIC contrib-libs-cxxsupp yutil - connector-api-protos + api-service-protos + connector-api-common contrib-libs-protobuf ) target_proto_messages(providers-generic-proto PRIVATE diff --git a/ydb/library/yql/providers/generic/proto/source.proto b/ydb/library/yql/providers/generic/proto/source.proto index 284af4530f..2f616be706 100644 --- a/ydb/library/yql/providers/generic/proto/source.proto +++ b/ydb/library/yql/providers/generic/proto/source.proto @@ -4,7 +4,8 @@ option cc_enable_arenas = true; package NYql.Generic; -import "ydb/library/yql/providers/generic/connector/api/protos/connector.proto"; +import "ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto"; +import "ydb/library/yql/providers/generic/connector/api/common/data_source.proto"; message TSource { // Token to access database diff --git a/ydb/library/yql/providers/generic/proto/ya.make b/ydb/library/yql/providers/generic/proto/ya.make index e757b582b8..350d192c15 100644 --- a/ydb/library/yql/providers/generic/proto/ya.make +++ b/ydb/library/yql/providers/generic/proto/ya.make @@ -3,7 +3,8 @@ PROTO_LIBRARY() ONLY_TAGS(CPP_PROTO) PEERDIR( - ydb/library/yql/providers/generic/connector/api/protos + ydb/library/yql/providers/generic/connector/api/service/protos + ydb/library/yql/providers/generic/connector/api/common ) SRCS( diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h index 36a555f252..3d6f36d722 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.h +++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.h @@ -4,7 +4,7 @@ #include <ydb/library/yql/providers/common/config/yql_setting.h> #include <ydb/library/yql/providers/common/db_id_async_resolver/db_async_resolver.h> #include <ydb/library/yql/providers/common/proto/gateways_config.pb.h> -#include <ydb/library/yql/providers/generic/connector/api/protos/connector.pb.h> +#include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h> #include <ydb/library/yql/utils/log/log.h> namespace NYql { |