aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authortsmax2004 <tsmax2004@yandex-team.com>2023-11-20 14:36:18 +0300
committertsmax2004 <tsmax2004@yandex-team.com>2023-11-20 17:31:38 +0300
commit712b15e09a5871f2676e04cb160afdf09b3217f3 (patch)
treee8078c233dc5f1def3bd585b39f7482e99cfd4dc
parent76b3103bf2a60ab088bbb580003cfe737cca4d34 (diff)
downloadydb-712b15e09a5871f2676e04cb160afdf09b3217f3.tar.gz
YQ Connector:new interface of query construct
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go35
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter.go54
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter_test.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/query_executor.go32
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter.go53
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter_test.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go10
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go3
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/rdbms/mock.go2
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/predicate_builder.go75
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/query_builder.go40
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go6
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/sql.go11
-rw-r--r--ydb/library/yql/providers/generic/connector/app/server/utils/ya.make1
16 files changed, 131 insertions, 197 deletions
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go
deleted file mode 100644
index a3b469d58c..0000000000
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/query_executor.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package clickhouse
-
-import (
- "context"
- "fmt"
-
- "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils"
- api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos"
-)
-
-type queryExecutor struct {
-}
-
-func (qm queryExecutor) DescribeTable(ctx context.Context, conn utils.Connection, request *api_service_protos.TDescribeTableRequest) (utils.Rows, error) {
- out, err := conn.Query(
- ctx,
- "SELECT name, type FROM system.columns WHERE table = ? and database = ?",
- request.Table,
- request.DataSourceInstance.Database,
- )
-
- if err != nil {
- return nil, fmt.Errorf("query: %w", err)
- }
-
- if err := out.Err(); err != nil {
- return nil, fmt.Errorf("rows err: %w", err)
- }
-
- return out, nil
-}
-
-func NewQueryExecutor() utils.QueryExecutor {
- return queryExecutor{}
-}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter.go
index 4f87ce1c3a..d2788a20d7 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter.go
@@ -1,22 +1,17 @@
package clickhouse
import (
- "fmt"
- "strings"
-
"github.com/ydb-platform/ydb-go-genproto/protos/Ydb"
- "github.com/ydb-platform/ydb/library/go/core/log"
"github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils"
api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos"
)
-type sqlFormatter struct {
-}
+var _ utils.SQLFormatter = (*sqlFormatter)(nil)
-type predicateBuilderFeatures struct {
+type sqlFormatter struct {
}
-func (f predicateBuilderFeatures) SupportsType(typeID Ydb.Type_PrimitiveTypeId) bool {
+func (f *sqlFormatter) supportsType(typeID Ydb.Type_PrimitiveTypeId) bool {
switch typeID {
case Ydb.Type_BOOL:
return true
@@ -45,23 +40,23 @@ func (f predicateBuilderFeatures) SupportsType(typeID Ydb.Type_PrimitiveTypeId)
}
}
-func (f predicateBuilderFeatures) SupportsConstantValueExpression(t *Ydb.Type) bool {
+func (f *sqlFormatter) supportsConstantValueExpression(t *Ydb.Type) bool {
switch v := t.Type.(type) {
case *Ydb.Type_TypeId:
- return f.SupportsType(v.TypeId)
+ return f.supportsType(v.TypeId)
case *Ydb.Type_OptionalType:
- return f.SupportsConstantValueExpression(v.OptionalType.Item)
+ return f.supportsConstantValueExpression(v.OptionalType.Item)
default:
return false
}
}
-func (f predicateBuilderFeatures) SupportsExpression(expression *api_service_protos.TExpression) bool {
+func (f sqlFormatter) SupportsPushdownExpression(expression *api_service_protos.TExpression) bool {
switch e := expression.Payload.(type) {
case *api_service_protos.TExpression_Column:
return true
case *api_service_protos.TExpression_TypedValue:
- return f.SupportsConstantValueExpression(e.TypedValue.Type)
+ return f.supportsConstantValueExpression(e.TypedValue.Type)
case *api_service_protos.TExpression_ArithmeticalExpression:
return false
case *api_service_protos.TExpression_Null:
@@ -71,31 +66,20 @@ func (f predicateBuilderFeatures) SupportsExpression(expression *api_service_pro
}
}
-func (formatter sqlFormatter) FormatRead(logger log.Logger, selectReq *api_service_protos.TSelect) (string, error) {
- var sb strings.Builder
-
- selectPart, err := utils.FormatSelectColumns(selectReq.What, selectReq.GetFrom().GetTable(), true)
- if err != nil {
- return "", fmt.Errorf("failed to format select statement: %w", err)
- }
+func (f sqlFormatter) GetDescribeTableQuery(request *api_service_protos.TDescribeTableRequest) (string, []any) {
+ query := "SELECT name, type FROM system.columns WHERE table = ? and database = ?"
+ args := []any{request.Table, request.DataSourceInstance.Database}
- sb.WriteString(selectPart)
-
- if selectReq.Where != nil {
- var features predicateBuilderFeatures
-
- clause, err := utils.FormatWhereClause(selectReq.Where, features)
- if err != nil {
- logger.Error("Failed to format WHERE clause", log.Error(err), log.String("where", selectReq.Where.String()))
- } else {
- sb.WriteString(" ")
- sb.WriteString(clause)
- }
- }
+ return query, args
+}
- query := sb.String()
+func (f sqlFormatter) GetPlaceholder(_ int) string {
+ return "?"
+}
- return query, nil
+func (f sqlFormatter) SanitiseIdentifier(ident string) string {
+ // TODO: sanitise
+ return ident
}
func NewSQLFormatter() utils.SQLFormatter {
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter_test.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter_test.go
index 5b6f264d98..05a78854de 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter_test.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/sql_formatter_test.go
@@ -330,7 +330,7 @@ func TestSQLFormatter(t *testing.T) {
tc := tc
t.Run(tc.testName, func(t *testing.T) {
- output, err := formatter.FormatRead(logger, tc.selectReq)
+ output, _, err := utils.MakeReadSplitQuery(logger, formatter, tc.selectReq)
require.Equal(t, tc.output, output)
if tc.err != nil {
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make
index a08bd57be7..46287aade5 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/ya.make
@@ -3,7 +3,6 @@ GO_LIBRARY()
SRCS(
connection_manager.go
doc.go
- query_executor.go
sql_formatter.go
type_mapper.go
)
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/query_executor.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/query_executor.go
deleted file mode 100644
index e894bfb3c7..0000000000
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/query_executor.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package postgresql
-
-import (
- "context"
- "fmt"
-
- "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils"
- api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos"
-)
-
-type queryExecutor struct {
-}
-
-func (qm queryExecutor) DescribeTable(ctx context.Context, conn utils.Connection, request *api_service_protos.TDescribeTableRequest) (utils.Rows, error) {
- schema := request.GetDataSourceInstance().GetPgOptions().GetSchema()
- out, err := conn.Query(
- ctx,
- "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2",
- request.Table,
- schema,
- )
-
- if err != nil {
- return nil, fmt.Errorf("connection query: %w", err)
- }
-
- return out, nil
-}
-
-func NewQueryExecutor() utils.QueryExecutor {
- return queryExecutor{}
-}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter.go
index fcaad0520b..ef59c7b182 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter.go
@@ -2,21 +2,18 @@ package postgresql
import (
"fmt"
- "strings"
"github.com/ydb-platform/ydb-go-genproto/protos/Ydb"
- "github.com/ydb-platform/ydb/library/go/core/log"
"github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/app/server/utils"
api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos"
)
-type sqlFormatter struct {
-}
+var _ utils.SQLFormatter = (*sqlFormatter)(nil)
-type predicateBuilderFeatures struct {
+type sqlFormatter struct {
}
-func (f predicateBuilderFeatures) SupportsType(typeID Ydb.Type_PrimitiveTypeId) bool {
+func (f *sqlFormatter) supportsType(typeID Ydb.Type_PrimitiveTypeId) bool {
switch typeID {
case Ydb.Type_BOOL:
return true
@@ -45,23 +42,23 @@ func (f predicateBuilderFeatures) SupportsType(typeID Ydb.Type_PrimitiveTypeId)
}
}
-func (f predicateBuilderFeatures) SupportsConstantValueExpression(t *Ydb.Type) bool {
+func (f *sqlFormatter) supportsConstantValueExpression(t *Ydb.Type) bool {
switch v := t.Type.(type) {
case *Ydb.Type_TypeId:
- return f.SupportsType(v.TypeId)
+ return f.supportsType(v.TypeId)
case *Ydb.Type_OptionalType:
- return f.SupportsConstantValueExpression(v.OptionalType.Item)
+ return f.supportsConstantValueExpression(v.OptionalType.Item)
default:
return false
}
}
-func (f predicateBuilderFeatures) SupportsExpression(expression *api_service_protos.TExpression) bool {
+func (f sqlFormatter) SupportsPushdownExpression(expression *api_service_protos.TExpression) bool {
switch e := expression.Payload.(type) {
case *api_service_protos.TExpression_Column:
return true
case *api_service_protos.TExpression_TypedValue:
- return f.SupportsConstantValueExpression(e.TypedValue.Type)
+ return f.supportsConstantValueExpression(e.TypedValue.Type)
case *api_service_protos.TExpression_ArithmeticalExpression:
return false
case *api_service_protos.TExpression_Null:
@@ -71,31 +68,21 @@ func (f predicateBuilderFeatures) SupportsExpression(expression *api_service_pro
}
}
-func (formatter sqlFormatter) FormatRead(logger log.Logger, selectReq *api_service_protos.TSelect) (string, error) {
- var sb strings.Builder
-
- selectPart, err := utils.FormatSelectColumns(selectReq.What, selectReq.GetFrom().GetTable(), true)
- if err != nil {
- return "", fmt.Errorf("failed to format select statement: %w", err)
- }
-
- sb.WriteString(selectPart)
-
- if selectReq.Where != nil {
- var features predicateBuilderFeatures
+func (f sqlFormatter) GetDescribeTableQuery(request *api_service_protos.TDescribeTableRequest) (string, []any) {
+ schema := request.GetDataSourceInstance().GetPgOptions().GetSchema()
+ query := "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2"
+ args := []any{request.Table, schema}
- clause, err := utils.FormatWhereClause(selectReq.Where, features)
- if err != nil {
- logger.Error("Failed to format WHERE clause", log.Error(err), log.String("where", selectReq.Where.String()))
- } else {
- sb.WriteString(" ")
- sb.WriteString(clause)
- }
- }
+ return query, args
+}
- query := sb.String()
+func (f sqlFormatter) GetPlaceholder(n int) string {
+ return fmt.Sprintf("$%d", n)
+}
- return query, nil
+func (f sqlFormatter) SanitiseIdentifier(ident string) string {
+ // TODO: sanitise
+ return ident
}
func NewSQLFormatter() utils.SQLFormatter {
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter_test.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter_test.go
index 16cd042fc4..1322dc04ea 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter_test.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/sql_formatter_test.go
@@ -330,7 +330,7 @@ func TestSQLFormatter(t *testing.T) {
tc := tc
t.Run(tc.testName, func(t *testing.T) {
- output, err := formatter.FormatRead(logger, tc.selectReq)
+ output, _, err := utils.MakeReadSplitQuery(logger, formatter, tc.selectReq)
require.Equal(t, tc.output, output)
if tc.err != nil {
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make b/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make
index a08bd57be7..46287aade5 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/ya.make
@@ -3,7 +3,6 @@ GO_LIBRARY()
SRCS(
connection_manager.go
doc.go
- query_executor.go
sql_formatter.go
type_mapper.go
)
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go
index 914a418133..b8f74335df 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go
@@ -13,7 +13,6 @@ import (
type handlerImpl struct {
typeMapper utils.TypeMapper
sqlFormatter utils.SQLFormatter
- queryBuilder utils.QueryExecutor
connectionManager utils.ConnectionManager
logger log.Logger
}
@@ -23,6 +22,8 @@ func (h *handlerImpl) DescribeTable(
logger log.Logger,
request *api_service_protos.TDescribeTableRequest,
) (*api_service_protos.TDescribeTableResponse, error) {
+ query, args := utils.MakeDescribeTableQuery(logger, h.sqlFormatter, request)
+
conn, err := h.connectionManager.Make(ctx, logger, request.DataSourceInstance)
if err != nil {
return nil, fmt.Errorf("make connection: %w", err)
@@ -30,7 +31,7 @@ func (h *handlerImpl) DescribeTable(
defer h.connectionManager.Release(logger, conn)
- rows, err := h.queryBuilder.DescribeTable(ctx, conn, request)
+ rows, err := conn.Query(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("query builder error: %w", err)
}
@@ -72,7 +73,7 @@ func (h *handlerImpl) doReadSplit(
split *api_service_protos.TSplit,
sink paging.Sink,
) error {
- query, err := h.sqlFormatter.FormatRead(logger, split.Select)
+ query, args, err := utils.MakeReadSplitQuery(logger, h.sqlFormatter, split.Select)
if err != nil {
return fmt.Errorf("make read split query: %w", err)
}
@@ -84,7 +85,7 @@ func (h *handlerImpl) doReadSplit(
defer h.connectionManager.Release(logger, conn)
- rows, err := conn.Query(ctx, query)
+ rows, err := conn.Query(ctx, query, args...)
if err != nil {
return fmt.Errorf("query '%s' error: %w", query, err)
}
@@ -136,7 +137,6 @@ func newHandler(
return &handlerImpl{
logger: logger,
sqlFormatter: preset.sqlFormatter,
- queryBuilder: preset.queryExecutor,
connectionManager: preset.connectionManager,
typeMapper: preset.typeMapper,
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
index 33560cebea..a1d5fd1b96 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
@@ -12,7 +12,6 @@ import (
type handlerPreset struct {
sqlFormatter utils.SQLFormatter
- queryExecutor utils.QueryExecutor
connectionManager utils.ConnectionManager
typeMapper utils.TypeMapper
}
@@ -44,13 +43,11 @@ func NewHandlerFactory(qlf utils.QueryLoggerFactory) HandlerFactory {
return &handlerFactoryImpl{
clickhouse: handlerPreset{
sqlFormatter: clickhouse.NewSQLFormatter(),
- queryExecutor: clickhouse.NewQueryExecutor(),
connectionManager: clickhouse.NewConnectionManager(connManagerCfg),
typeMapper: clickhouse.NewTypeMapper(),
},
postgresql: handlerPreset{
sqlFormatter: postgresql.NewSQLFormatter(),
- queryExecutor: postgresql.NewQueryExecutor(),
connectionManager: postgresql.NewConnectionManager(connManagerCfg),
typeMapper: postgresql.NewTypeMapper(),
},
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/mock.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/mock.go
index dbe05948dd..73f63b22ad 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/mock.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/mock.go
@@ -42,7 +42,6 @@ var _ HandlerFactory = (*HandlerFactoryMock)(nil)
type HandlerFactoryMock struct {
SQLFormatter utils.SQLFormatter
- QueryExecutor utils.QueryExecutor
ConnectionManager utils.ConnectionManager
TypeMapper utils.TypeMapper
}
@@ -52,7 +51,6 @@ func (m *HandlerFactoryMock) Make(logger log.Logger, dataSourceType api_common.E
logger,
&handlerPreset{
sqlFormatter: m.SQLFormatter,
- queryExecutor: m.QueryExecutor,
connectionManager: m.ConnectionManager,
typeMapper: m.TypeMapper,
},
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/predicate_builder.go b/ydb/library/yql/providers/generic/connector/app/server/utils/predicate_builder.go
index b763a9a265..6ed1aebad8 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/predicate_builder.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/predicate_builder.go
@@ -8,12 +8,7 @@ import (
api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos"
)
-type PredicateBuilderFeatures interface {
- // Support for high level expression (without subexpressions, they are checked separately)
- SupportsExpression(expression *api_service_protos.TExpression) bool
-}
-
-func FormatValue(value *Ydb.TypedValue) (string, error) {
+func formatValue(value *Ydb.TypedValue) (string, error) {
switch v := value.Value.Value.(type) {
case *Ydb.Value_BoolValue:
return fmt.Sprintf("%t", v.BoolValue), nil
@@ -34,15 +29,15 @@ func FormatValue(value *Ydb.TypedValue) (string, error) {
}
}
-func FormatColumn(col string) (string, error) {
+func formatColumn(col string) (string, error) {
return col, nil
}
-func FormatNull(n *api_service_protos.TExpression_TNull) (string, error) {
+func formatNull(n *api_service_protos.TExpression_TNull) (string, error) {
return "NULL", nil
}
-func FormatArithmeticalExpression(expression *api_service_protos.TExpression_TArithmeticalExpression, features PredicateBuilderFeatures) (string, error) {
+func formatArithmeticalExpression(formatter SQLFormatter, expression *api_service_protos.TExpression_TArithmeticalExpression) (string, error) {
var operation string
switch op := expression.Operation; op {
@@ -68,12 +63,12 @@ func FormatArithmeticalExpression(expression *api_service_protos.TExpression_TAr
err error
)
- left, err = FormatExpression(expression.LeftValue, features)
+ left, err = formatExpression(formatter, expression.LeftValue)
if err != nil {
return "", fmt.Errorf("failed to format left argument: %w", err)
}
- right, err = FormatExpression(expression.RightValue, features)
+ right, err = formatExpression(formatter, expression.RightValue)
if err != nil {
return "", fmt.Errorf("failed to format right argument: %w", err)
}
@@ -81,26 +76,26 @@ func FormatArithmeticalExpression(expression *api_service_protos.TExpression_TAr
return fmt.Sprintf("(%s%s%s)", left, operation, right), nil
}
-func FormatExpression(expression *api_service_protos.TExpression, features PredicateBuilderFeatures) (string, error) {
- if !features.SupportsExpression(expression) {
+func formatExpression(formatter SQLFormatter, expression *api_service_protos.TExpression) (string, error) {
+ if !formatter.SupportsPushdownExpression(expression) {
return "", ErrUnsupportedExpression
}
switch e := expression.Payload.(type) {
case *api_service_protos.TExpression_Column:
- return FormatColumn(e.Column)
+ return formatColumn(e.Column)
case *api_service_protos.TExpression_TypedValue:
- return FormatValue(e.TypedValue)
+ return formatValue(e.TypedValue)
case *api_service_protos.TExpression_ArithmeticalExpression:
- return FormatArithmeticalExpression(e.ArithmeticalExpression, features)
+ return formatArithmeticalExpression(formatter, e.ArithmeticalExpression)
case *api_service_protos.TExpression_Null:
- return FormatNull(e.Null)
+ return formatNull(e.Null)
default:
return "", fmt.Errorf("%w, type: %T", ErrUnimplementedExpression, e)
}
}
-func FormatComparison(comparison *api_service_protos.TPredicate_TComparison, features PredicateBuilderFeatures) (string, error) {
+func formatComparison(formatter SQLFormatter, comparison *api_service_protos.TPredicate_TComparison) (string, error) {
var operation string
switch op := comparison.Operation; op {
@@ -126,12 +121,12 @@ func FormatComparison(comparison *api_service_protos.TPredicate_TComparison, fea
err error
)
- left, err = FormatExpression(comparison.LeftValue, features)
+ left, err = formatExpression(formatter, comparison.LeftValue)
if err != nil {
return "", fmt.Errorf("failed to format left argument: %w", err)
}
- right, err = FormatExpression(comparison.RightValue, features)
+ right, err = formatExpression(formatter, comparison.RightValue)
if err != nil {
return "", fmt.Errorf("failed to format right argument: %w", err)
}
@@ -139,8 +134,8 @@ func FormatComparison(comparison *api_service_protos.TPredicate_TComparison, fea
return fmt.Sprintf("(%s%s%s)", left, operation, right), nil
}
-func FormatNegation(negation *api_service_protos.TPredicate_TNegation, features PredicateBuilderFeatures) (string, error) {
- pred, err := FormatPredicate(negation.Operand, features, false)
+func formatNegation(formatter SQLFormatter, negation *api_service_protos.TPredicate_TNegation) (string, error) {
+ pred, err := formatPredicate(formatter, negation.Operand, false)
if err != nil {
return "", fmt.Errorf("failed to format NOT statement: %w", err)
}
@@ -148,7 +143,7 @@ func FormatNegation(negation *api_service_protos.TPredicate_TNegation, features
return fmt.Sprintf("(NOT %s)", pred), nil
}
-func FormatConjunction(conjunction *api_service_protos.TPredicate_TConjunction, features PredicateBuilderFeatures, topLevel bool) (string, error) {
+func formatConjunction(formatter SQLFormatter, conjunction *api_service_protos.TPredicate_TConjunction, topLevel bool) (string, error) {
var (
sb strings.Builder
succeeded int32 = 0
@@ -158,7 +153,7 @@ func FormatConjunction(conjunction *api_service_protos.TPredicate_TConjunction,
)
for _, predicate := range conjunction.Operands {
- statement, err = FormatPredicate(predicate, features, false)
+ statement, err = formatPredicate(formatter, predicate, false)
if err != nil {
if !topLevel {
return "", fmt.Errorf("failed to format AND statement: %w", err)
@@ -193,7 +188,7 @@ func FormatConjunction(conjunction *api_service_protos.TPredicate_TConjunction,
return sb.String(), nil
}
-func FormatDisjunction(disjunction *api_service_protos.TPredicate_TDisjunction, features PredicateBuilderFeatures) (string, error) {
+func formatDisjunction(formatter SQLFormatter, disjunction *api_service_protos.TPredicate_TDisjunction) (string, error) {
var (
sb strings.Builder
cnt int32 = 0
@@ -203,7 +198,7 @@ func FormatDisjunction(disjunction *api_service_protos.TPredicate_TDisjunction,
)
for _, predicate := range disjunction.Operands {
- statement, err = FormatPredicate(predicate, features, false)
+ statement, err = formatPredicate(formatter, predicate, false)
if err != nil {
return "", fmt.Errorf("failed to format OR statement: %w", err)
} else {
@@ -236,8 +231,8 @@ func FormatDisjunction(disjunction *api_service_protos.TPredicate_TDisjunction,
return sb.String(), nil
}
-func FormatIsNull(isNull *api_service_protos.TPredicate_TIsNull, features PredicateBuilderFeatures) (string, error) {
- statement, err := FormatExpression(isNull.Value, features)
+func formatIsNull(formatter SQLFormatter, isNull *api_service_protos.TPredicate_TIsNull) (string, error) {
+ statement, err := formatExpression(formatter, isNull.Value)
if err != nil {
return "", fmt.Errorf("failed to format IS NULL statement: %w", err)
}
@@ -245,8 +240,8 @@ func FormatIsNull(isNull *api_service_protos.TPredicate_TIsNull, features Predic
return fmt.Sprintf("(%s IS NULL)", statement), nil
}
-func FormatIsNotNull(isNotNull *api_service_protos.TPredicate_TIsNotNull, features PredicateBuilderFeatures) (string, error) {
- statement, err := FormatExpression(isNotNull.Value, features)
+func formatIsNotNull(formatter SQLFormatter, isNotNull *api_service_protos.TPredicate_TIsNotNull) (string, error) {
+ statement, err := formatExpression(formatter, isNotNull.Value)
if err != nil {
return "", fmt.Errorf("failed to format IS NOT NULL statement: %w", err)
}
@@ -254,33 +249,33 @@ func FormatIsNotNull(isNotNull *api_service_protos.TPredicate_TIsNotNull, featur
return fmt.Sprintf("(%s IS NOT NULL)", statement), nil
}
-func FormatPredicate(predicate *api_service_protos.TPredicate, features PredicateBuilderFeatures, topLevel bool) (string, error) {
+func formatPredicate(formatter SQLFormatter, predicate *api_service_protos.TPredicate, topLevel bool) (string, error) {
switch p := predicate.Payload.(type) {
case *api_service_protos.TPredicate_Negation:
- return FormatNegation(p.Negation, features)
+ return formatNegation(formatter, p.Negation)
case *api_service_protos.TPredicate_Conjunction:
- return FormatConjunction(p.Conjunction, features, topLevel)
+ return formatConjunction(formatter, p.Conjunction, topLevel)
case *api_service_protos.TPredicate_Disjunction:
- return FormatDisjunction(p.Disjunction, features)
+ return formatDisjunction(formatter, p.Disjunction)
case *api_service_protos.TPredicate_IsNull:
- return FormatIsNull(p.IsNull, features)
+ return formatIsNull(formatter, p.IsNull)
case *api_service_protos.TPredicate_IsNotNull:
- return FormatIsNotNull(p.IsNotNull, features)
+ return formatIsNotNull(formatter, p.IsNotNull)
case *api_service_protos.TPredicate_Comparison:
- return FormatComparison(p.Comparison, features)
+ return formatComparison(formatter, p.Comparison)
case *api_service_protos.TPredicate_BoolExpression:
- return FormatExpression(p.BoolExpression.Value, features)
+ return formatExpression(formatter, p.BoolExpression.Value)
default:
return "", fmt.Errorf("%w, type: %T", ErrUnimplementedPredicateType, p)
}
}
-func FormatWhereClause(where *api_service_protos.TSelect_TWhere, features PredicateBuilderFeatures) (string, error) {
+func formatWhereClause(formatter SQLFormatter, where *api_service_protos.TSelect_TWhere) (string, error) {
if where.FilterTyped == nil {
return "", ErrUnimplemented
}
- formatted, err := FormatPredicate(where.FilterTyped, features, true)
+ formatted, err := formatPredicate(formatter, where.FilterTyped, true)
if err != nil {
return "", err
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/query_builder.go b/ydb/library/yql/providers/generic/connector/app/server/utils/query_builder.go
new file mode 100644
index 0000000000..5fad123291
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/query_builder.go
@@ -0,0 +1,40 @@
+package utils
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/ydb-platform/ydb/library/go/core/log"
+ api_service_protos "github.com/ydb-platform/ydb/ydb/library/yql/providers/generic/connector/libgo/service/protos"
+)
+
+func MakeDescribeTableQuery(logger log.Logger, formatter SQLFormatter, request *api_service_protos.TDescribeTableRequest) (string, []any) {
+ query, args := formatter.GetDescribeTableQuery(request)
+
+ return query, args
+}
+
+func MakeReadSplitQuery(logger log.Logger, formatter SQLFormatter, request *api_service_protos.TSelect) (string, []any, error) {
+ var sb strings.Builder
+
+ selectPart, err := formatSelectColumns(formatter, request.What, request.GetFrom().GetTable(), true)
+ if err != nil {
+ return "", nil, fmt.Errorf("failed to format select statement: %w", err)
+ }
+
+ sb.WriteString(selectPart)
+
+ if request.Where != nil {
+ clause, err := formatWhereClause(formatter, request.Where)
+ if err != nil {
+ logger.Error("Failed to format WHERE clause", log.Error(err), log.String("where", request.Where.String()))
+ } else {
+ sb.WriteString(" ")
+ sb.WriteString(clause)
+ }
+ }
+
+ query := sb.String()
+
+ return query, nil, nil
+}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go b/ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go
index ebf5e7d0c1..352da3fab0 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/select_helpers.go
@@ -23,7 +23,7 @@ func SelectWhatToYDBTypes(selectWhat *api_service_protos.TSelect_TWhat) ([]*Ydb.
return ydbTypes, nil
}
-func SelectWhatToYDBColumns(selectWhat *api_service_protos.TSelect_TWhat) ([]*Ydb.Column, error) {
+func selectWhatToYDBColumns(selectWhat *api_service_protos.TSelect_TWhat) ([]*Ydb.Column, error) {
var columns []*Ydb.Column
for i, item := range selectWhat.Items {
@@ -38,7 +38,7 @@ func SelectWhatToYDBColumns(selectWhat *api_service_protos.TSelect_TWhat) ([]*Yd
return columns, nil
}
-func FormatSelectColumns(selectWhat *api_service_protos.TSelect_TWhat, tableName string, fakeZeroOnEmptyColumnsSet bool) (string, error) {
+func formatSelectColumns(formatter SQLFormatter, selectWhat *api_service_protos.TSelect_TWhat, tableName string, fakeZeroOnEmptyColumnsSet bool) (string, error) {
// SELECT $columns FROM $from
if tableName == "" {
return "", ErrEmptyTableName
@@ -48,7 +48,7 @@ func FormatSelectColumns(selectWhat *api_service_protos.TSelect_TWhat, tableName
sb.WriteString("SELECT ")
- columns, err := SelectWhatToYDBColumns(selectWhat)
+ columns, err := selectWhatToYDBColumns(selectWhat)
if err != nil {
return "", fmt.Errorf("convert Select.What.Items to Ydb.Columns: %w", err)
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/sql.go b/ydb/library/yql/providers/generic/connector/app/server/utils/sql.go
index 51d7c2da76..d4b015da3a 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/sql.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/sql.go
@@ -30,10 +30,11 @@ type ConnectionManagerBase struct {
QueryLoggerFactory QueryLoggerFactory
}
-type QueryExecutor interface {
- DescribeTable(ctx context.Context, conn Connection, request *api_service_protos.TDescribeTableRequest) (Rows, error)
-}
-
type SQLFormatter interface {
- FormatRead(logger log.Logger, selectReq *api_service_protos.TSelect) (string, error)
+ GetDescribeTableQuery(request *api_service_protos.TDescribeTableRequest) (string, []any)
+ GetPlaceholder(n int) string
+ SanitiseIdentifier(ident string) string
+
+ // Support for high level expression (without subexpressions, they are checked separately)
+ SupportsPushdownExpression(expression *api_service_protos.TExpression) bool
}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make b/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make
index a2c30d45ee..28eafde38a 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/ya.make
@@ -10,6 +10,7 @@ SRCS(
logger.go
predicate_builder.go
protobuf.go
+ query_builder.go
select_helpers.go
sql.go
sql_mock.go