diff options
author | vitalyisaev <vitalyisaev@ydb.tech> | 2023-10-09 10:33:30 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@ydb.tech> | 2023-10-09 11:00:20 +0300 |
commit | 9ca0bdf5c33c563bbb56937d8cd47191d2aa2911 (patch) | |
tree | 6f037499504a62bb1eb52d26dcdef0ae6e35f93d | |
parent | 3603a96b240210bb1f45276faa3b387631405f1c (diff) | |
download | ydb-9ca0bdf5c33c563bbb56937d8cd47191d2aa2911.tar.gz |
YQ Connector: enable more linters
Включаем дополнительные линтеры, доступные в Аркадии
24 files changed, 69 insertions, 5 deletions
diff --git a/ydb/library/yql/providers/generic/connector/app/client/client.go b/ydb/library/yql/providers/generic/connector/app/client/client.go index 1318625ba8..6a90271e1b 100644 --- a/ydb/library/yql/providers/generic/connector/app/client/client.go +++ b/ydb/library/yql/providers/generic/connector/app/client/client.go @@ -144,8 +144,10 @@ func describeTable( if utils.IsSuccess(resp.Error) { logger.Debug("DescribeTable", log.String("response", resp.String())) + return resp.Schema, nil } + logger.Error("DescribeTable", log.String("response", resp.String())) return nil, utils.NewSTDErrorFromAPIError(resp.Error) @@ -195,6 +197,7 @@ func listSplits( if !utils.IsSuccess(resp.Error) { logger.Error("ListSplits", log.String("response", resp.String())) + return splits, utils.NewSTDErrorFromAPIError(resp.Error) } diff --git a/ydb/library/yql/providers/generic/connector/app/client/doc.go b/ydb/library/yql/providers/generic/connector/app/client/doc.go new file mode 100644 index 0000000000..60f37e4bae --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/app/client/doc.go @@ -0,0 +1,3 @@ +// Package client contains code of a simple CLI application +// that can be used for server debugging. +package client diff --git a/ydb/library/yql/providers/generic/connector/app/client/ya.make b/ydb/library/yql/providers/generic/connector/app/client/ya.make index c3855e7300..47bd0180b0 100644 --- a/ydb/library/yql/providers/generic/connector/app/client/ya.make +++ b/ydb/library/yql/providers/generic/connector/app/client/ya.make @@ -1,5 +1,8 @@ GO_LIBRARY() -SRCS(client.go) +SRCS( + client.go + doc.go +) END() diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go index 66cf6233ae..4d9cd45d5f 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go +++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go @@ -23,7 +23,17 @@ type Connection struct { func (c Connection) Query(ctx context.Context, query string, args ...any) (utils.Rows, error) { c.logger.Dump(query, args...) - return c.DB.QueryContext(ctx, query, args...) + + rows, err := c.DB.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query context: %w", err) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("rows err: %w", err) + } + + return rows, nil } var _ utils.ConnectionManager[*Connection] = (*connectionManager)(nil) @@ -56,6 +66,7 @@ func (c *connectionManager) Make( }, DialContext: func(ctx context.Context, addr string) (net.Conn, error) { var d net.Dialer + return d.DialContext(ctx, "tcp", addr) }, Debug: true, diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/doc.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/doc.go new file mode 100644 index 0000000000..aeeb567bdb --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/doc.go @@ -0,0 +1,2 @@ +// Package clickhouse contains code specific for ClickHouse database. +package clickhouse diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go index f15f8176f3..0c09fd84f1 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go +++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go @@ -23,6 +23,10 @@ func (qm queryExecutor) DescribeTable(ctx context.Context, conn *Connection, req return nil, fmt.Errorf("query: %w", err) } + if err := out.Err(); err != nil { + return nil, fmt.Errorf("rows err: %w", err) + } + return out, nil } diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go index 9483fefdd9..eb0ce8197d 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go +++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/type_mapper.go @@ -174,7 +174,6 @@ func (tm typeMapper) AddRowToArrowIPCStreaming(ydbTypes []*Ydb.Type, acceptors [ } for i, ydbType := range ydbTypes { - switch t := ydbType.Type.(type) { case *Ydb.Type_TypeId: if err := tm.appendValueToBuilder(t.TypeId, acceptors[i], builders[i], false); err != nil { @@ -203,30 +202,38 @@ func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils optional bool, ) error { var value IN + if optional { cast := acceptor.(**IN) if *cast == nil { builder.AppendNull() + return nil } + value = **cast } else { + //nolint:forcetypeassert value = *acceptor.(*IN) } var converter CONV + out, err := converter.Convert(value) if err != nil { if errors.Is(err, utils.ErrValueOutOfTypeBounds) { // TODO: write warning to logger builder.AppendNull() + return nil } return fmt.Errorf("convert value %v: %w", value, err) } + //nolint:forcetypeassert builder.(AB).Append(out) + return nil } @@ -237,6 +244,7 @@ func (typeMapper) appendValueToBuilder( optional bool, ) error { var err error + switch typeID { case Ydb.Type_BOOL: err = appendValueToArrowBuilder[bool, uint8, *array.Uint8Builder, utils.BoolConverter](acceptor, builder, optional) diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make index a04c18a0aa..6925db2cf7 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make +++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make @@ -2,6 +2,7 @@ GO_LIBRARY() SRCS( connection_manager.go + doc.go query_executor.go type_mapper.go ) diff --git a/ydb/library/yql/providers/generic/connector/app/server/doc.go b/ydb/library/yql/providers/generic/connector/app/server/doc.go new file mode 100644 index 0000000000..dbdca1745b --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/app/server/doc.go @@ -0,0 +1,4 @@ +// Package server containes the code base of Connector GRPC service - +// the component of YDB's Federated Query system providing the unified interface +// to the external data sources. +package server diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go index 9bbf0d8032..c776bc96ed 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go +++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go @@ -20,6 +20,7 @@ type rows struct { func (r rows) Close() error { r.Rows.Close() + return nil } @@ -36,6 +37,7 @@ func (c Connection) Query(ctx context.Context, query string, args ...any) (utils c.logger.Dump(query, args...) out, err := c.Conn.Query(ctx, query, args...) + return rows{Rows: out}, err } @@ -74,6 +76,7 @@ func (c *connectionManager) Make( if err != nil { return nil, fmt.Errorf("parse connection config template: %w", err) } + connCfg.Database = dsi.Database connCfg.Host = dsi.GetEndpoint().GetHost() connCfg.Port = uint16(dsi.GetEndpoint().GetPort()) diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/doc.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/doc.go new file mode 100644 index 0000000000..9df86d415c --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/doc.go @@ -0,0 +1,2 @@ +// Package postgresql contains code specific for PostgreSQL database. +package postgresql diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go index c637ee0f34..363714089c 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go +++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/type_mapper.go @@ -120,6 +120,7 @@ func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils ) error { if !valid { builder.AppendNull() + return nil } @@ -130,6 +131,7 @@ func appendValueToArrowBuilder[IN utils.ValueType, OUT utils.ValueType, AB utils if errors.Is(err, utils.ErrValueOutOfTypeBounds) { // TODO: logger ? builder.AppendNull() + return nil } diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make b/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make index a04c18a0aa..6925db2cf7 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make +++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make @@ -2,6 +2,7 @@ GO_LIBRARY() SRCS( connection_manager.go + doc.go query_executor.go type_mapper.go ) diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/doc.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/doc.go new file mode 100644 index 0000000000..4261e7ef33 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/doc.go @@ -0,0 +1,3 @@ +// Package rdbms contains generalized abstractions suitable for use with +// any relational database. +package rdbms diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go index 759f620f3f..6adca6351e 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go +++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go @@ -25,7 +25,6 @@ func (hf *HandlerFactory) Make( logger log.Logger, dataSourceType api_common.EDataSourceKind, ) (Handler, error) { - switch dataSourceType { case api_common.EDataSourceKind_CLICKHOUSE: return newHandler[*clickhouse.Connection](logger, &hf.clickhouse), nil diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go index e2e2ea1d5f..5d56a5b9ee 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go +++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/schema_builder.go @@ -35,6 +35,7 @@ func (sb *schemaBuilder) addColumn(columnName, columnType string) error { } sb.items = append(sb.items, item) + return nil } diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make b/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make index a8a933ea14..e9a5b03e24 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make +++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/ya.make @@ -1,6 +1,7 @@ GO_LIBRARY() SRCS( + doc.go handler.go handler_factory.go schema_builder.go diff --git a/ydb/library/yql/providers/generic/connector/app/server/server.go b/ydb/library/yql/providers/generic/connector/app/server/server.go index ec3f4a33bc..0055934493 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/server.go +++ b/ydb/library/yql/providers/generic/connector/app/server/server.go @@ -83,11 +83,13 @@ func (s *Server) ListSplits(request *api_service_protos.TListSplitsRequest, stre for _, slct := range request.Selects { if err := s.doListSplitsHandleSelect(stream, slct, &totalSplits); err != nil { logger.Error("request handling failed", log.Error(err)) + return err } } logger.Info("request handling finished", log.Int("total_splits", totalSplits)) + return nil } @@ -105,6 +107,7 @@ func (s *Server) doListSplitsHandleSelect( for _, split := range resp.Splits { logger.Debug("responding split", log.Int("split_id", *totalSplits), log.String("split", split.Select.String())) + *totalSplits++ } @@ -126,6 +129,7 @@ func (s *Server) doListSplitsResponse( if err := stream.Send(response); err != nil { logger.Error("send channel failed", log.Error(err)) + return err } @@ -143,6 +147,7 @@ func (s *Server) ReadSplits(request *api_service_protos.TReadSplitsRequest, stre if err := stream.Send(response); err != nil { logger.Error("send channel failed", log.Error(err)) + return err } } @@ -160,6 +165,7 @@ func (s *Server) ReadSplits(request *api_service_protos.TReadSplitsRequest, stre if err := stream.Send(response); err != nil { logger.Error("send channel failed", log.Error(err)) + return err } } @@ -245,6 +251,7 @@ func (s *Server) makeOptions() ([]grpc.ServerOption, error) { s.logger.Info("server will use TLS connections") s.logger.Info("reading key pair", log.String("cert", s.cfg.Tls.Cert), log.String("key", s.cfg.Tls.Key)) + creds, err := credentials.NewServerTLSFromFile(s.cfg.Tls.Cert, s.cfg.Tls.Key) if err != nil { return nil, fmt.Errorf("new server TLS from file: %w", err) diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go b/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go index c78e631fcc..1cc176d5c6 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go +++ b/ydb/library/yql/providers/generic/connector/app/server/utils/arrow_helpers.go @@ -126,5 +126,6 @@ func primitiveTypeToArrow(typeID Ydb.Type_PrimitiveTypeId, column *Ydb.Column, a default: return arrow.Field{}, nil, fmt.Errorf("register type '%v': %w", typeID, ErrDataTypeNotSupported) } + return field, builder, nil } diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go b/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go new file mode 100644 index 0000000000..9925b62fe7 --- /dev/null +++ b/ydb/library/yql/providers/generic/connector/app/server/utils/doc.go @@ -0,0 +1,2 @@ +// Package utils contains various helpers and utility functions +package utils diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go index 279524353c..575de397ae 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go +++ b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go @@ -15,6 +15,7 @@ import ( ) // TODO: it's better to do this in GRPC middleware + func AnnotateLogger(logger log.Logger, method string, dsi *api_common.TDataSourceInstance) log.Logger { logger = log.With(logger, log.String("method", method)) @@ -85,6 +86,7 @@ type QueryLoggerFactory struct { func NewQueryLoggerFactory(cfg *config.TLoggerConfig) QueryLoggerFactory { enabled := cfg.GetEnableSqlQueryLogging() + return QueryLoggerFactory{enableQueryLogging: enabled} } diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make b/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make index 0e57385587..f675fae924 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make +++ b/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make @@ -6,6 +6,7 @@ SRCS( columnar_buffer_factory.go connection_manager.go converters.go + doc.go endpoint.go errors.go logger.go diff --git a/ydb/library/yql/providers/generic/connector/app/server/validate.go b/ydb/library/yql/providers/generic/connector/app/server/validate.go index 06005d7900..486d11ea0e 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/validate.go +++ b/ydb/library/yql/providers/generic/connector/app/server/validate.go @@ -22,7 +22,6 @@ func ValidateDescribeTableRequest(logger log.Logger, request *api_service_protos } func ValidateListSplitsRequest(logger log.Logger, request *api_service_protos.TListSplitsRequest) error { - if len(request.Selects) == 0 { return fmt.Errorf("empty select list: %w", utils.ErrInvalidRequest) } diff --git a/ydb/library/yql/providers/generic/connector/app/server/ya.make b/ydb/library/yql/providers/generic/connector/app/server/ya.make index 500a6363a2..7bd36d8d8f 100644 --- a/ydb/library/yql/providers/generic/connector/app/server/ya.make +++ b/ydb/library/yql/providers/generic/connector/app/server/ya.make @@ -2,6 +2,7 @@ GO_LIBRARY() SRCS( cmd.go + doc.go config.go server.go validate.go |