.logging_queries

add query logging

9 files changed, 94 insertions, 29 deletions

diff --git a/ydb/library/yql/providers/generic/connector/app/config/server.pb.go b/ydb/library/yql/providers/generic/connector/app/config/server.pb.go
index fb8f895b6a4..9ac7186ed7b 100644
--- a/ydb/library/yql/providers/generic/connector/app/config/server.pb.go
+++ b/ydb/library/yql/providers/generic/connector/app/config/server.pb.go
@@ -273,6 +273,8 @@ type TLoggerConfig struct {
 
 	// Level of logging
 	LogLevel ELogLevel `protobuf:"varint,1,opt,name=log_level,json=logLevel,proto3,enum=NYql.Connector.App.Config.ELogLevel" json:"log_level,omitempty"`
+	// Is logging of queries enabled
+	EnableSqlQueryLogging bool `protobuf:"varint,2,opt,name=enable_sql_query_logging,json=enableSqlQueryLogging,proto3" json:"enable_sql_query_logging,omitempty"`
 }
 
 func (x *TLoggerConfig) Reset() {
@@ -314,6 +316,13 @@ func (x *TLoggerConfig) GetLogLevel() ELogLevel {
 	return ELogLevel_TRACE
 }
 
+func (x *TLoggerConfig) GetEnableSqlQueryLogging() bool {
+	if x != nil {
+		return x.EnableSqlQueryLogging
+	}
+	return false
+}
+
 var File_ydb_library_yql_providers_generic_connector_app_config_server_proto protoreflect.FileDescriptor
 
 var file_ydb_library_yql_providers_generic_connector_app_config_server_proto_rawDesc = []byte{
@@ -351,22 +360,26 @@ var file_ydb_library_yql_providers_generic_connector_app_config_server_proto_raw
 	0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x4a, 0x04, 0x08,
 	0x01, 0x10, 0x02, 0x22, 0x26, 0x0a, 0x10, 0x54, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
 	0x61, 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x22, 0x52, 0x0a, 0x0d, 0x54,
-	0x4c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x41, 0x0a, 0x09,
-	0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32,
-	0x24, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72,
-	0x2e, 0x41, 0x70, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x45, 0x4c, 0x6f, 0x67,
-	0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x2a,
-	0x4b, 0x0a, 0x09, 0x45, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x09, 0x0a, 0x05,
-	0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47,
-	0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04,
-	0x57, 0x41, 0x52, 0x4e, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10,
-	0x04, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x05, 0x42, 0x49, 0x5a, 0x47,
-	0x61, 0x2e, 0x79, 0x61, 0x6e, 0x64, 0x65, 0x78, 0x2d, 0x74, 0x65, 0x61, 0x6d, 0x2e, 0x72, 0x75,
-	0x2f, 0x79, 0x64, 0x62, 0x2f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, 0x6c,
-	0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72,
-	0x69, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x70,
-	0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x0d,
+	0x54, 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x41, 0x0a,
+	0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e,
+	0x32, 0x24, 0x2e, 0x4e, 0x59, 0x71, 0x6c, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f,
+	0x72, 0x2e, 0x41, 0x70, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x45, 0x4c, 0x6f,
+	0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c,
+	0x12, 0x37, 0x0a, 0x18, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x71, 0x6c, 0x5f, 0x71,
+	0x75, 0x65, 0x72, 0x79, 0x5f, 0x6c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x08, 0x52, 0x15, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x71, 0x6c, 0x51, 0x75, 0x65,
+	0x72, 0x79, 0x4c, 0x6f, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x2a, 0x4b, 0x0a, 0x09, 0x45, 0x4c, 0x6f,
+	0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10,
+	0x00, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04,
+	0x49, 0x4e, 0x46, 0x4f, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x03,
+	0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x46,
+	0x41, 0x54, 0x41, 0x4c, 0x10, 0x05, 0x42, 0x49, 0x5a, 0x47, 0x61, 0x2e, 0x79, 0x61, 0x6e, 0x64,
+	0x65, 0x78, 0x2d, 0x74, 0x65, 0x61, 0x6d, 0x2e, 0x72, 0x75, 0x2f, 0x79, 0x64, 0x62, 0x2f, 0x6c,
+	0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x2f, 0x79, 0x71, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69,
+	0x64, 0x65, 0x72, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x2f, 0x63, 0x6f, 0x6e,
+	0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x70, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69,
+	0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
diff --git a/ydb/library/yql/providers/generic/connector/app/config/server.proto b/ydb/library/yql/providers/generic/connector/app/config/server.proto
index aa71ea677fc..d179cc08e73 100644
--- a/ydb/library/yql/providers/generic/connector/app/config/server.proto
+++ b/ydb/library/yql/providers/generic/connector/app/config/server.proto
@@ -39,6 +39,8 @@ message TServerReadLimit {
 message TLoggerConfig {
     // Level of logging
     ELogLevel log_level = 1;
+    // Is logging of queries enabled
+    bool enable_sql_query_logging = 2;
 }
 
 // ELogLevel enumerates standard levels of logging
diff --git a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go
index 5cdbf69db7b..66cf6233ae4 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/clickhouse/connection_manager.go
@@ -18,15 +18,18 @@ var _ utils.Connection = (*Connection)(nil)
 
 type Connection struct {
 	*sql.DB
+	logger utils.QueryLogger
 }
 
 func (c Connection) Query(ctx context.Context, query string, args ...any) (utils.Rows, error) {
+	c.logger.Dump(query, args...)
 	return c.DB.QueryContext(ctx, query, args...)
 }
 
 var _ utils.ConnectionManager[*Connection] = (*connectionManager)(nil)
 
 type connectionManager struct {
+	utils.ConnectionManagerBase
 	// TODO: cache of connections, remove unused connections with TTL
 }
 
@@ -92,13 +95,15 @@ func (c *connectionManager) Make(
 	conn.SetMaxOpenConns(maxOpenConns)
 	conn.SetConnMaxLifetime(time.Hour)
 
-	return &Connection{DB: conn}, nil
+	queryLogger := c.QueryLoggerFactory.Make(logger)
+
+	return &Connection{DB: conn, logger: queryLogger}, nil
 }
 
 func (c *connectionManager) Release(logger log.Logger, conn *Connection) {
 	utils.LogCloserError(logger, conn, "close clickhouse connection")
 }
 
-func NewConnectionManager() utils.ConnectionManager[*Connection] {
-	return &connectionManager{}
+func NewConnectionManager(cfg utils.ConnectionManagerBase) utils.ConnectionManager[*Connection] {
+	return &connectionManager{ConnectionManagerBase: cfg}
 }
diff --git a/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go
index 5d5fa46ffbb..9bbf0d8032a 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/postgresql/connection_manager.go
@@ -25,6 +25,7 @@ func (r rows) Close() error {
 
 type Connection struct {
 	*pgx.Conn
+	logger utils.QueryLogger
 }
 
 func (c Connection) Close() error {
@@ -32,6 +33,8 @@ func (c Connection) Close() error {
 }
 
 func (c Connection) Query(ctx context.Context, query string, args ...any) (utils.Rows, error) {
+	c.logger.Dump(query, args...)
+
 	out, err := c.Conn.Query(ctx, query, args...)
 	return rows{Rows: out}, err
 }
@@ -39,12 +42,13 @@ func (c Connection) Query(ctx context.Context, query string, args ...any) (utils
 var _ utils.ConnectionManager[*Connection] = (*connectionManager)(nil)
 
 type connectionManager struct {
+	utils.ConnectionManagerBase
 	// TODO: cache of connections, remove unused connections with TTL
 }
 
 func (c *connectionManager) Make(
 	ctx context.Context,
-	_ log.Logger,
+	logger log.Logger,
 	dsi *api_common.TDataSourceInstance,
 ) (*Connection, error) {
 	if dsi.GetCredentials().GetBasic() == nil {
@@ -85,13 +89,15 @@ func (c *connectionManager) Make(
 		return nil, fmt.Errorf("open connection: %w", err)
 	}
 
-	return &Connection{conn}, nil
+	queryLogger := c.QueryLoggerFactory.Make(logger)
+
+	return &Connection{conn, queryLogger}, nil
 }
 
 func (c *connectionManager) Release(logger log.Logger, conn *Connection) {
 	utils.LogCloserError(logger, conn, "close posgresql connection")
 }
 
-func NewConnectionManager() utils.ConnectionManager[*Connection] {
-	return &connectionManager{}
+func NewConnectionManager(cfg utils.ConnectionManagerBase) utils.ConnectionManager[*Connection] {
+	return &connectionManager{ConnectionManagerBase: cfg}
 }
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go
index 83ce1f4bb55..772ba72f63e 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler.go
@@ -143,8 +143,6 @@ func (h *handlerImpl[CONN]) ReadSplit(
 
 	query := sb.String()
 
-	logger.Debug("execute query", log.String("query", query))
-
 	rows, err := conn.Query(ctx, query)
 	if err != nil {
 		return fmt.Errorf("query '%s' error: %w", query, err)
diff --git a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
index e7d03426b15..759f620f3f7 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/rdbms/handler_factory.go
@@ -36,16 +36,20 @@ func (hf *HandlerFactory) Make(
 	}
 }
 
-func NewHandlerFactory() *HandlerFactory {
+func NewHandlerFactory(qlf utils.QueryLoggerFactory) *HandlerFactory {
+	connManagerCfg := utils.ConnectionManagerBase{
+		QueryLoggerFactory: qlf,
+	}
+
 	return &HandlerFactory{
 		clickhouse: preset[*clickhouse.Connection]{
 			queryExecutor:     clickhouse.NewQueryExecutor(),
-			connectionManager: clickhouse.NewConnectionManager(),
+			connectionManager: clickhouse.NewConnectionManager(connManagerCfg),
 			typeMapper:        clickhouse.NewTypeMapper(),
 		},
 		postgresql: preset[*postgresql.Connection]{
 			queryExecutor:     postgresql.NewQueryExecutor(),
-			connectionManager: postgresql.NewConnectionManager(),
+			connectionManager: postgresql.NewConnectionManager(connManagerCfg),
 			typeMapper:        postgresql.NewTypeMapper(),
 		},
 	}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/server.go b/ydb/library/yql/providers/generic/connector/app/server/server.go
index 78030729848..ec3f4a33bce 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/server.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/server.go
@@ -262,8 +262,10 @@ func newServer(
 	logger log.Logger,
 	cfg *config.TServerConfig,
 ) (*Server, error) {
+	queryLoggerFactory := utils.NewQueryLoggerFactory(cfg.Logger)
+
 	return &Server{
-		handlerFactory: rdbms.NewHandlerFactory(),
+		handlerFactory: rdbms.NewHandlerFactory(queryLoggerFactory),
 		columnarBufferFactory: utils.NewColumnarBufferFactory(
 			memory.DefaultAllocator,
 			utils.NewReadLimiterFactory(cfg.ReadLimit),
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/connection_manager.go b/ydb/library/yql/providers/generic/connector/app/server/utils/connection_manager.go
index 46738960b8b..bf3c2e520c1 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/connection_manager.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/connection_manager.go
@@ -23,3 +23,7 @@ type ConnectionManager[CONN any] interface {
 	Make(ctx context.Context, logger log.Logger, dataSourceInstance *api_common.TDataSourceInstance) (CONN, error)
 	Release(logger log.Logger, conn CONN)
 }
+
+type ConnectionManagerBase struct {
+	QueryLoggerFactory QueryLoggerFactory
+}
diff --git a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
index f3a281406cb..279524353ce 100644
--- a/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
+++ b/ydb/library/yql/providers/generic/connector/app/server/utils/logger.go
@@ -79,6 +79,37 @@ func DumpReadSplitsResponse(logger log.Logger, resp *api_service_protos.TReadSpl
 	}
 }
 
+type QueryLoggerFactory struct {
+	enableQueryLogging bool
+}
+
+func NewQueryLoggerFactory(cfg *config.TLoggerConfig) QueryLoggerFactory {
+	enabled := cfg.GetEnableSqlQueryLogging()
+	return QueryLoggerFactory{enableQueryLogging: enabled}
+}
+
+func (f *QueryLoggerFactory) Make(logger log.Logger) QueryLogger {
+	return QueryLogger{logger: logger, enabled: f.enableQueryLogging}
+}
+
+type QueryLogger struct {
+	logger  log.Logger
+	enabled bool
+}
+
+func (ql *QueryLogger) Dump(query string, args ...any) {
+	if !ql.enabled {
+		return
+	}
+
+	logFields := []log.Field{log.String("query", query)}
+	if len(args) > 0 {
+		logFields = append(logFields, log.Any("args", args))
+	}
+
+	ql.logger.Debug("execute SQL query", logFields...)
+}
+
 func convertToZapLogLevel(lvl config.ELogLevel) zapcore.Level {
 	switch lvl {
 	case config.ELogLevel_TRACE, config.ELogLevel_DEBUG: