aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvitalyisaev <vitalyisaev@ydb.tech>2023-10-09 10:33:30 +0300
committervitalyisaev <vitalyisaev@ydb.tech>2023-10-09 11:00:20 +0300
commit9ca0bdf5c33c563bbb56937d8cd47191d2aa2911 (patch)
tree6f037499504a62bb1eb52d26dcdef0ae6e35f93d
parent3603a96b240210bb1f45276faa3b387631405f1c (diff)
downloadydb-9ca0bdf5c33c563bbb56937d8cd47191d2aa2911.tar.gz
YQ Connector: enable more linters
Включаем дополнительные линтеры, доступные в Аркадии
-rw-r--r--ydb/library/yql/providers/generic/connector/app/client/client.go3
-rw-r--r--ydb/library/yql/providers/generic/connector/app/client/doc.go3
-rw-r--r--ydb/library/yql/providers/generic/connector/app/client/ya.make5
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go13
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/doc.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go4
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go10
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/doc.go4
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go3
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/doc.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/rdbms/doc.go3
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/server.go7
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/doc.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/logger.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/validate.go1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/ya.make1
24 files changed, 69 insertions, 5 deletions
diff --git a/ydb/library/yql/providers/generic/connector/app/client/client.go b/ydb/library/yql/providers/generic/connector/app/client/client.go
index 1318625ba8..6a90271e1b 100644
--- a/ydb/library/yql/providers/generic/connector/app/client/client.go
+++ b/ydb/library/yql/providers/generic/connector/app/client/client.go
@@ -144,8 +144,10 @@ func describeTable(
if utils.IsSuccess(resp.Error) {
logger.Debug("DescribeTable", log.String("response", resp.String()))
+
return resp.Schema, nil
}
+
logger.Error("DescribeTable", log.String("response", resp.String()))
return nil, utils.NewSTDErrorFromAPIError(resp.Error)
@@ -195,6 +197,7 @@ func listSplits(
if !utils.IsSuccess(resp.Error) {
logger.Error("ListSplits", log.String("response", resp.String()))
+
return splits, utils.NewSTDErrorFromAPIError(resp.Error)
}
diff --git a/ydb/library/yql/providers/generic/connector/app/client/doc.go b/ydb/library/yql/providers/generic/connector/app/client/doc.go
new file mode 100644
index 0000000000..60f37e4bae
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/app/client/doc.go
@@ -0,0 +1,3 @@
+// Package client contains code of a simple CLI application
+// that can be used for server debugging.
+package client
diff --git a/ydb/library/yql/providers/generic/connector/app/client/ya.make b/ydb/library/yql/providers/generic/connector/app/client/ya.make
index c3855e7300..47bd0180b0 100644
--- a/ydb/library/yql/providers/generic/connector/app/client/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/client/ya.make
@@ -1,5 +1,8 @@
GO_LIBRARY()
-SRCS(client.go)
+SRCS(
+ client.go
+ doc.go
+)
END()
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go
index 66cf6233ae..4d9cd45d5f 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go
@@ -23,7 +23,17 @@ type Connection struct {
func (c Connection) Query(ctx context.Context, query string, args ...any) (utils.Rows, error) {
c.logger.Dump(query, args...)
- return c.DB.QueryContext(ctx, query, args...)
+
+ rows, err := c.DB.QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, fmt.Errorf("query context: %w", err)
+ }
+
+ if err := rows.Err(); err != nil {
+ return nil, fmt.Errorf("rows err: %w", err)
+ }
+
+ return rows, nil
}
var _ utils.ConnectionManager[*Connection] = (*connectionManager)(nil)
@@ -56,6 +66,7 @@ func (c *connectionManager) Make(
},
DialContext: func(ctx context.Context, addr string) (net.Conn, error) {
var d net.Dialer
+
return d.DialContext(ctx, "tcp", addr)
},
Debug: true,
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/doc.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/doc.go
new file mode 100644
index 0000000000..aeeb567bdb
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/doc.go
@@ -0,0 +1,2 @@
+// Package clickhouse contains code specific for ClickHouse database.
+package clickhouse
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go
index f15f8176f3..0c09fd84f1 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go
@@ -23,6 +23,10 @@ func (qm queryExecutor) DescribeTable(ctx context.Context, conn *Connection, req
return nil, fmt.Errorf("query: %w", err)
}
+ if err := out.Err(); err != nil {
+ return nil, fmt.Errorf("rows err: %w", err)
+ }
+
return out, nil
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go
index 9483fefdd9..eb0ce8197d 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go
@@ -174,7 +174,6 @@ func (tm typeMapper) AddRowToArrowIPCStreaming(ydbTypes []*Ydb.Type, acceptors [
}
for i, ydbType := range ydbTypes {
-
switch t := ydbType.Type.(type) {
case *Ydb.Type_TypeId:
if err := tm.appendValueToBuilder(t.TypeId, acceptors[i], builders[i], false); err != nil {
@@ -203,30 +202,38 @@ func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils
optional bool,
) error {
var value IN
+
if optional {
cast := acceptor.(**IN)
if *cast == nil {
builder.AppendNull()
+
return nil
}
+
value = **cast
} else {
+ //nolint:forcetypeassert
value = *acceptor.(*IN)
}
var converter CONV
+
out, err := converter.Convert(value)
if err != nil {
if errors.Is(err, utils.ErrValueOutOfTypeBounds) {
// TODO: write warning to logger
builder.AppendNull()
+
return nil
}
return fmt.Errorf("convert value %v: %w", value, err)
}
+ //nolint:forcetypeassert
builder.(AB).Append(out)
+
return nil
}
@@ -237,6 +244,7 @@ func (typeMapper) appendValueToBuilder(
optional bool,
) error {
var err error
+
switch typeID {
case Ydb.Type_BOOL:
err = appendValueToArrowBuilder[bool, uint8, *array.Uint8Builder, utils.BoolConverter](acceptor, builder, optional)
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make
index a04c18a0aa..6925db2cf7 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make
@@ -2,6 +2,7 @@ GO_LIBRARY()
SRCS(
connection_manager.go
+ doc.go
query_executor.go
type_mapper.go
)
diff --git a/ydb/library/yql/providers/generic/connector/app/server/doc.go b/ydb/library/yql/providers/generic/connector/app/server/doc.go
new file mode 100644
index 0000000000..dbdca1745b
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/app/server/doc.go
@@ -0,0 +1,4 @@
+// Package server containes the code base of Connector GRPC service -
+// the component of YDB's Federated Query system providing the unified interface
+// to the external data sources.
+package server
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go
index 9bbf0d8032..c776bc96ed 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go
@@ -20,6 +20,7 @@ type rows struct {
func (r rows) Close() error {
r.Rows.Close()
+
return nil
}
@@ -36,6 +37,7 @@ func (c Connection) Query(ctx context.Context, query string, args ...any) (utils
c.logger.Dump(query, args...)
out, err := c.Conn.Query(ctx, query, args...)
+
return rows{Rows: out}, err
}
@@ -74,6 +76,7 @@ func (c *connectionManager) Make(
if err != nil {
return nil, fmt.Errorf("parse connection config template: %w", err)
}
+
connCfg.Database = dsi.Database
connCfg.Host = dsi.GetEndpoint().GetHost()
connCfg.Port = uint16(dsi.GetEndpoint().GetPort())
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/doc.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/doc.go
new file mode 100644
index 0000000000..9df86d415c
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/doc.go
@@ -0,0 +1,2 @@
+// Package postgresql contains code specific for PostgreSQL database.
+package postgresql
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go
index c637ee0f34..363714089c 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go
@@ -120,6 +120,7 @@ func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils
) error {
if !valid {
builder.AppendNull()
+
return nil
}
@@ -130,6 +131,7 @@ func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils
if errors.Is(err, utils.ErrValueOutOfTypeBounds) {
// TODO: logger ?
builder.AppendNull()
+
return nil
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make b/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make
index a04c18a0aa..6925db2cf7 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make
@@ -2,6 +2,7 @@ GO_LIBRARY()
SRCS(
connection_manager.go
+ doc.go
query_executor.go
type_mapper.go
)
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/doc.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/doc.go
new file mode 100644
index 0000000000..4261e7ef33
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/doc.go
@@ -0,0 +1,3 @@
+// Package rdbms contains generalized abstractions suitable for use with
+// any relational database.
+package rdbms
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
index 759f620f3f..6adca6351e 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
@@ -25,7 +25,6 @@ func (hf *HandlerFactory) Make(
logger log.Logger,
dataSourceType api_common.EDataSourceKind,
) (Handler, error) {
-
switch dataSourceType {
case api_common.EDataSourceKind_CLICKHOUSE:
return newHandler[*clickhouse.Connection](logger, &hf.clickhouse), nil
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go
index e2e2ea1d5f..5d56a5b9ee 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go
@@ -35,6 +35,7 @@ func (sb *schemaBuilder) addColumn(columnName, columnType string) error {
}
sb.items = append(sb.items, item)
+
return nil
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make b/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make
index a8a933ea14..e9a5b03e24 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make
@@ -1,6 +1,7 @@
GO_LIBRARY()
SRCS(
+ doc.go
handler.go
handler_factory.go
schema_builder.go
diff --git a/ydb/library/yql/providers/generic/connector/app/server/server.go b/ydb/library/yql/providers/generic/connector/app/server/server.go
index ec3f4a33bc..0055934493 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/server.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/server.go
@@ -83,11 +83,13 @@ func (s *Server) ListSplits(request *api_service_protos.TListSplitsRequest, stre
for _, slct := range request.Selects {
if err := s.doListSplitsHandleSelect(stream, slct, &totalSplits); err != nil {
logger.Error("request handling failed", log.Error(err))
+
return err
}
}
logger.Info("request handling finished", log.Int("total_splits", totalSplits))
+
return nil
}
@@ -105,6 +107,7 @@ func (s *Server) doListSplitsHandleSelect(
for _, split := range resp.Splits {
logger.Debug("responding split", log.Int("split_id", *totalSplits), log.String("split", split.Select.String()))
+
*totalSplits++
}
@@ -126,6 +129,7 @@ func (s *Server) doListSplitsResponse(
if err := stream.Send(response); err != nil {
logger.Error("send channel failed", log.Error(err))
+
return err
}
@@ -143,6 +147,7 @@ func (s *Server) ReadSplits(request *api_service_protos.TReadSplitsRequest, stre
if err := stream.Send(response); err != nil {
logger.Error("send channel failed", log.Error(err))
+
return err
}
}
@@ -160,6 +165,7 @@ func (s *Server) ReadSplits(request *api_service_protos.TReadSplitsRequest, stre
if err := stream.Send(response); err != nil {
logger.Error("send channel failed", log.Error(err))
+
return err
}
}
@@ -245,6 +251,7 @@ func (s *Server) makeOptions() ([]grpc.ServerOption, error) {
s.logger.Info("server will use TLS connections")
s.logger.Info("reading key pair", log.String("cert", s.cfg.Tls.Cert), log.String("key", s.cfg.Tls.Key))
+
creds, err := credentials.NewServerTLSFromFile(s.cfg.Tls.Cert, s.cfg.Tls.Key)
if err != nil {
return nil, fmt.Errorf("new server TLS from file: %w", err)
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go b/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go
index c78e631fcc..1cc176d5c6 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go
@@ -126,5 +126,6 @@ func primitiveTypeToArrow(typeID Ydb.Type_PrimitiveTypeId, column *Ydb.Column, a
default:
return arrow.Field{}, nil, fmt.Errorf("register type '%v': %w", typeID, ErrDataTypeNotSupported)
}
+
return field, builder, nil
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go b/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go
new file mode 100644
index 0000000000..9925b62fe7
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go
@@ -0,0 +1,2 @@
+// Package utils contains various helpers and utility functions
+package utils
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
index 279524353c..575de397ae 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
@@ -15,6 +15,7 @@ import (
)
// TODO: it's better to do this in GRPC middleware
+
func AnnotateLogger(logger log.Logger, method string, dsi *api_common.TDataSourceInstance) log.Logger {
logger = log.With(logger, log.String("method", method))
@@ -85,6 +86,7 @@ type QueryLoggerFactory struct {
func NewQueryLoggerFactory(cfg *config.TLoggerConfig) QueryLoggerFactory {
enabled := cfg.GetEnableSqlQueryLogging()
+
return QueryLoggerFactory{enableQueryLogging: enabled}
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make b/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make
index 0e57385587..f675fae924 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make
@@ -6,6 +6,7 @@ SRCS(
columnar_buffer_factory.go
connection_manager.go
converters.go
+ doc.go
endpoint.go
errors.go
logger.go
diff --git a/ydb/library/yql/providers/generic/connector/app/server/validate.go b/ydb/library/yql/providers/generic/connector/app/server/validate.go
index 06005d7900..486d11ea0e 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/validate.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/validate.go
@@ -22,7 +22,6 @@ func ValidateDescribeTableRequest(logger log.Logger, request *api_service_protos
}
func ValidateListSplitsRequest(logger log.Logger, request *api_service_protos.TListSplitsRequest) error {
-
if len(request.Selects) == 0 {
return fmt.Errorf("empty select list: %w", utils.ErrInvalidRequest)
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/ya.make b/ydb/library/yql/providers/generic/connector/app/server/ya.make
index 500a6363a2..7bd36d8d8f 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/ya.make
@@ -2,6 +2,7 @@ GO_LIBRARY()
SRCS(
cmd.go
+ doc.go
config.go
server.go
validate.go