YQ Connector:export ydb/library/yql/providers/generic/connector/debug to github

34 files changed, 785 insertions, 0 deletions

diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/config/z_log_disable.xml b/ydb/library/yql/providers/generic/connector/debug/clickhouse/config/z_log_disable.xml
new file mode 100644
index 0000000000..eb11758f05
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/clickhouse/config/z_log_disable.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <asynchronous_metric_log remove="1"/>
+    <metric_log remove="1"/>
+    <query_thread_log remove="1" />  
+    <query_log remove="1" />
+    <query_views_log remove="1" />
+    <part_log remove="1"/>
+    <session_log remove="1"/>
+    <text_log remove="1" />
+    <trace_log remove="1"/>
+    <crash_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <zookeeper_log remove="1"/>
+</clickhouse>
diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/init/init_db.sh b/ydb/library/yql/providers/generic/connector/debug/clickhouse/init/init_db.sh
new file mode 100644
index 0000000000..2f6f3b957c
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/clickhouse/init/init_db.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+set -e
+
+clickhouse client -n <<-EOSQL
+    DROP TABLE IF EXISTS dqrun.example_1;
+    CREATE TABLE dqrun.example_1 (id Int32, col1 String, col2 Int32) ENGINE = MergeTree ORDER BY id;
+    INSERT INTO dqrun.example_1 (*) VALUES (1, 'ch_example_1_a', 10);
+    INSERT INTO dqrun.example_1 (*) VALUES (2, 'ch_example_1_b', 20);
+    INSERT INTO dqrun.example_1 (*) VALUES (3, 'ch_example_1_c', 30);
+    INSERT INTO dqrun.example_1 (*) VALUES (4, 'ch_example_1_d', 40);
+    INSERT INTO dqrun.example_1 (*) VALUES (5, 'ch_example_1_e', 50);
+EOSQL
+
+clickhouse client -n <<-EOSQL
+    DROP TABLE IF EXISTS dqrun.date_time;
+    CREATE TABLE dqrun.date_time (
+        col_date Date,
+        col_datetime DateTime,
+        col_datetime_zone DateTime('Europe/Moscow'),
+        col_date64 DateTime64(3),
+        col_date64_zone DateTime64(3, 'Asia/Istanbul')
+    ) ENGINE = TinyLog;
+    INSERT INTO dqrun.date_time (*) VALUES (now(), now(), now(), now(), now());
+    INSERT INTO dqrun.date_time (*) VALUES ('2004-01-10', '2004-01-10 00:05:00', '2004-01-10 00:05:00', '2004-01-10T00:05:00.321', '2004-01-10T00:05:00.321');
+    INSERT INTO dqrun.date_time (*) VALUES ('1950-01-10', '1950-01-10 00:05:00', '1950-01-10 00:05:00', '2100-01-10T00:05:00.321', '2004-01-10T00:05:00.321');
+EOSQL
+
+clickhouse client -n <<-EOSQL
+    DROP TABLE IF EXISTS dqrun.primitives;
+    CREATE TABLE dqrun.primitives (
+        col_01_boolean Boolean,
+        col_02_int8 Int8,
+        col_03_uint8 UInt8,
+        col_04_int16 Int16,
+        col_05_uint16 UInt16,
+        col_06_int32 Int32,
+        col_07_uint32 UInt32,
+        col_08_int64 Int64,
+        col_09_uint64 UInt64,
+        col_10_float32 Float32,
+        col_11_float64 Float64,
+        col_12_string String,
+        col_13_string FixedString(13),
+        col_14_date Date,
+        col_16_datetime DateTime,
+        col_17_date64 DateTime64(3)
+    ) ENGINE = TinyLog;
+    INSERT INTO dqrun.primitives (*) VALUES (True, 2, 3, 4, 5, 6, 7, 8, 9, 10.10, 11.11, 'az', 'az', now(), now(), '1988-11-20 12:55:08.123');
+    INSERT INTO dqrun.primitives (*) VALUES (False, -2, 3, -4, 5, -6, 7, -8, 9, -10.10, -11.11, 'буки', 'буки', now(), now(), now());
+EOSQL
+
+clickhouse client -n <<-EOSQL
+    DROP TABLE IF EXISTS dqrun.optional;
+    CREATE TABLE dqrun.optional (
+        col_01_boolean Nullable(Boolean),
+        col_02_int8 Nullable(Int8),
+        col_03_uint8 Nullable(UInt8),
+        col_04_int16 Nullable(Int16),
+        col_05_uint16 Nullable(UInt16),
+        col_06_int32 Nullable(Int32),
+        col_07_uint32 Nullable(UInt32),
+        col_08_int64 Nullable(Int64),
+        col_09_uint64 Nullable(UInt64),
+        col_10_float32 Nullable(Float32),
+        col_11_float64 Nullable(Float64),
+        col_12_string Nullable(String),
+        col_13_string Nullable(FixedString(13)),
+        col_14_date Nullable(Date),
+        col_15_datetime Nullable(DateTime),
+        col_16_date64 Nullable(DateTime64(3))
+    ) ENGINE = TinyLog;
+    INSERT INTO dqrun.optional (*) VALUES (True, 2, 3, 4, 5, 6, 7, 8, 9, 10.10, 11.11, 'az', 'az', now(), now(), now());
+    INSERT INTO dqrun.optional (*) VALUES (False, -2, 3, -4, 5, -6, 7, -8, 9, -10.10, -11.11, 'буки', 'буки', now(), now(), now());
+    INSERT INTO dqrun.optional (*) VALUES (NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
+EOSQL
+
+clickhouse client -n <<-EOSQL
+    DROP TABLE IF EXISTS dqrun.unsupported;
+    CREATE TABLE dqrun.unsupported (id Int32, col1 String, col2 UUID) ENGINE = TinyLog;
+    INSERT INTO dqrun.unsupported (*) VALUES (1, 'ch_az', generateUUIDv4());
+    INSERT INTO dqrun.unsupported (*) VALUES (2, 'ch_buki', generateUUIDv4());
+    INSERT INTO dqrun.unsupported (*) VALUES (3, 'ch_vedi', generateUUIDv4());
+    INSERT INTO dqrun.unsupported (*) VALUES (4, 'ch_glagol', generateUUIDv4());
+    INSERT INTO dqrun.unsupported (*) VALUES (5, 'ch_dobro', generateUUIDv4());
+EOSQL
+
+clickhouse client -n <<-EOSQL
+    DROP TABLE IF EXISTS dqrun.benchmark_1g;
+    CREATE TABLE dqrun.benchmark_1g (id UInt64, col Text) ENGINE = MergeTree ORDER BY id;
+    INSERT INTO dqrun.benchmark_1g SELECT *, randomPrintableASCII(randUniform(1024, 1024)) FROM numbers(1048576);
+EOSQL
diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/init_table_datetime.sh b/ydb/library/yql/providers/generic/connector/debug/clickhouse/init_table_datetime.sh
new file mode 100644
index 0000000000..48ceff8fde
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/clickhouse/init_table_datetime.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -ex
+
+URL='http://crab:qwerty12345@localhost:8123/?database=dqrun'
+
+SCRIPTS=(
+"DROP TABLE IF EXISTS date_time;"
+"CREATE TABLE date_time (
+    col_date Date,
+    col_date32 Date32,
+    col_datetime DateTime,
+    col_datetime64 DateTime64(3)
+) ENGINE = TinyLog;"
+"INSERT INTO date_time (*) VALUES ('1950-01-10', '1850-01-10', '1950-01-10 12:23:45', '1850-01-10 12:23:45.678');"
+"INSERT INTO date_time (*) VALUES ('1970-01-10', '1950-01-10', '1970-01-10 12:23:45', '1950-01-10 12:23:45.678');"
+"INSERT INTO date_time (*) VALUES ('2004-01-10', '2004-01-10', '2004-01-10 12:23:45', '2004-01-10 12:23:45.678');"
+"INSERT INTO date_time (*) VALUES ('2110-01-10', '2110-01-10', '2106-01-10 12:23:45', '2110-01-10 12:23:45.678');"
+"INSERT INTO date_time (*) VALUES ('2150-01-10', '2300-01-10', '2107-01-10 12:23:45', '2300-01-10 12:23:45.678');"
+)
+
+for ((i = 0; i < ${#SCRIPTS[@]}; i++))
+do
+    echo "${SCRIPTS[$i]}" | curl "${URL}" --data-binary @-
+done
+
+echo "SELECT * FROM date_time" | curl "${URL}" --data-binary @-
diff --git a/ydb/library/yql/providers/generic/connector/debug/clickhouse/maintain/show_table_size.sh b/ydb/library/yql/providers/generic/connector/debug/clickhouse/maintain/show_table_size.sh
new file mode 100755
index 0000000000..a81926c5fb
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/clickhouse/maintain/show_table_size.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -ex
+
+URL='http://admin:password@localhost:8123/?database=dqrun'
+
+#echo "SELECT table, formatReadableSize(sum(bytes)) as size, min(min_date) as min_date, max(max_date) as max_date
+#      FROM system.parts WHERE active GROUP BY table" | curl "${URL}" --data-binary @-
+
+echo "SELECT
+        database,
+        table,
+        formatReadableSize(sum(data_compressed_bytes) AS size) AS compressed,
+        formatReadableSize(sum(data_uncompressed_bytes) AS usize) AS uncompressed,
+        round(usize / size, 2) AS compr_rate,
+        sum(rows) AS rows,
+        count() AS part_count
+    FROM system.parts
+    WHERE (active = 1) AND (database LIKE '%') AND (table LIKE '%')
+    GROUP BY database, table ORDER BY size DESC;" | curl "${URL}" --data-binary @-
diff --git a/ydb/library/yql/providers/generic/connector/debug/config/client/ch.local.txt b/ydb/library/yql/providers/generic/connector/debug/config/client/ch.local.txt
new file mode 100644
index 0000000000..5cf862e32a
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/config/client/ch.local.txt
@@ -0,0 +1,19 @@
+endpoint {
+    host: "localhost"
+    port: 50051
+}
+
+data_source_instance {
+    kind: CLICKHOUSE
+    endpoint {
+        host: "localhost"
+        port: 9000
+    }
+    database: "dqrun"
+    credentials {
+        basic {
+            username: "crab"
+            password: "qwerty12345"
+        }
+    }
+}
diff --git a/ydb/library/yql/providers/generic/connector/debug/config/client/pg.cloud.txt b/ydb/library/yql/providers/generic/connector/debug/config/client/pg.cloud.txt
new file mode 100644
index 0000000000..e516a510bc
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/config/client/pg.cloud.txt
@@ -0,0 +1,23 @@
+endpoint {
+    host: "connector.yql-streaming.cloud.yandex.net"
+    port: 50051
+}
+
+data_source_instance {
+    kind: CLICKHOUSE
+    endpoint {
+        host: "rc1a-d6dv17lv47v5mcop.mdb.yandexcloud.net"
+        port: 9000
+    }
+    database: "dqrun"
+    credentials {
+        basic {
+            username: "admin"
+            password: "password"
+        }
+    }
+}
+
+tls {
+    ca: "/home/vitalyisaev/arcadia/kikimr/deployment/terraform/yc-crt/YandexInternalRootCA.crt"
+}
diff --git a/ydb/library/yql/providers/generic/connector/debug/config/client/pg.local.txt b/ydb/library/yql/providers/generic/connector/debug/config/client/pg.local.txt
new file mode 100644
index 0000000000..88ef9cd1e2
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/config/client/pg.local.txt
@@ -0,0 +1,19 @@
+endpoint {
+    host: "localhost"
+    port: 50051
+}
+
+data_source_instance {
+    kind: POSTGRESQL
+    endpoint {
+        host: "localhost"
+        port: 5432
+    }
+    database: "dqrun"
+    credentials {
+        basic {
+            username: "crab"
+            password: "qwerty12345"
+        }
+    }
+}
diff --git a/ydb/library/yql/providers/generic/connector/debug/config/server/server.no_tls.txt b/ydb/library/yql/providers/generic/connector/debug/config/server/server.no_tls.txt
new file mode 100644
index 0000000000..f2732c366f
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/config/server/server.no_tls.txt
@@ -0,0 +1,23 @@
+connector_server {
+    endpoint {
+        host: "0.0.0.0"
+        port: 50051
+    }
+}
+
+logger {
+    log_level: TRACE
+    enable_sql_query_logging: true
+}
+
+pprof_server {
+    endpoint {
+        host: "0.0.0.0"
+        port: 6060
+    }
+}
+
+paging {
+    bytes_per_page: 4194304
+    prefetch_queue_capacity: 2
+}
diff --git a/ydb/library/yql/providers/generic/connector/debug/config/server/server.tls.txt b/ydb/library/yql/providers/generic/connector/debug/config/server/server.tls.txt
new file mode 100644
index 0000000000..d5f3498a49
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/config/server/server.tls.txt
@@ -0,0 +1,15 @@
+endpoint {
+    host: "0.0.0.0"
+    port: 50051
+}
+
+tls {
+    ca: "/home/vitalyisaev/arcadia/junk/vitalyisaev/connectors/tls/ca.crt"
+    key: "/home/vitalyisaev/arcadia/junk/vitalyisaev/connectors/tls/server.key"
+    cert: "/home/vitalyisaev/arcadia/junk/vitalyisaev/connectors/tls/server.crt"
+}
+
+logger {
+    log_level: TRACE
+    enable_sql_query_logging: true
+}
diff --git a/ydb/library/yql/providers/generic/connector/debug/docker-compose.yaml b/ydb/library/yql/providers/generic/connector/debug/docker-compose.yaml
new file mode 100644
index 0000000000..5ce59dd548
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/docker-compose.yaml
@@ -0,0 +1,58 @@
+version: '3'
+
+services:
+
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    container_name: yq-connector-clickhouse
+    ports:
+      - '8123:8123'
+      - '9000:9000'
+      - '9009:9009'
+    environment:
+      CLICKHOUSE_DB: dqrun
+      CLICKHOUSE_USER: admin
+      CLICKHOUSE_PASSWORD: password
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - /tmp/yq-connector/clickhouse/data:/var/lib/clickhouse
+      - /tmp/yq-connector/clickhouse/logs:/var/log/clickhouse-server
+      - ./clickhouse/init:/docker-entrypoint-initdb.d
+      - ./clickhouse/config/z_log_disable.xml:/etc/clickhouse-server/config.d/z_log_disable.xml
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 262144
+        hard: 262144
+
+  postgresql:
+    image: postgres
+    container_name: yq-connector-postgresql
+    ports:
+      - '5432:5432'
+    environment:
+      POSTGRES_DB: dqrun
+      POSTGRES_USER: admin
+      POSTGRES_PASSWORD: password
+      PGDATA: /var/lib/postgresql/data/pgdata
+    volumes:
+      - /tmp/yq-connector/postgresql/data/:/var/lib/postgresql/data/
+      - ./postgresql/init:/docker-entrypoint-initdb.d
+
+  minio:
+    image: minio/minio:latest
+    container_name: yq-connector-minio
+    command: server --console-address ':9001' /data/
+    ports:
+      - '29000:9000'
+      - '29001:9001'
+    environment:
+      MINIO_ROOT_USER: admin
+      MINIO_ROOT_PASSWORD: password
+    volumes:
+      - /tmp/yq-connector/minio/data:/data
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:29000/minio/health/live']
+      interval: 30s
+      timeout: 20s
+      retries: 3
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.cloud.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.cloud.yql
new file mode 100644
index 0000000000..8bdfcff780
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.cloud.yql
@@ -0,0 +1 @@
+SELECT * FROM clickhouse_streaming.example_1;
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.local.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.local.yql
new file mode 100644
index 0000000000..3709b65b7a
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.local.yql
@@ -0,0 +1,4 @@
+-- SELECT col_15_date32 FROM rtmr_dev00_clickhouse_native.primitives;
+-- SELECT MIN(LEN(col)) FROM rtmr_dev00_clickhouse_native.benchmark_1g;
+-- SELECT MIN(LEN(col1)+ LEN(col2)) FROM rtmr_dev00_clickhouse_native.benchmark_2g;
+SELECT * FROM rtmr_dev00_clickhouse_native.example_1;
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.single.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.single.yql
new file mode 100644
index 0000000000..cb090ffc6d
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/ch.single.yql
@@ -0,0 +1,3 @@
+pragma UseBlocks;
+
+SELECT id, id * id AS id_square FROM rtmr_dev00_clickhouse.example_1;
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/debug.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/debug.yql
new file mode 100644
index 0000000000..4672cd3294
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/debug.yql
@@ -0,0 +1 @@
+SELECT col_14_date, col_16_datetime , col_17_date64 FROM rtmr_dev00_clickhouse.primitives;
\ No newline at end of file
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.cloud.conf b/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.cloud.conf
new file mode 100644
index 0000000000..51d3cabcb3
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.cloud.conf
@@ -0,0 +1,95 @@
+Generic {
+  Connector {
+    Endpoint {
+      host: "connector.yql-streaming.cloud.yandex.net"
+      port: 50051
+    }
+    UseSsl: true
+  }
+
+	ClusterMapping {
+    Kind: CLICKHOUSE,
+    Name: "clickhouse_streaming"
+    DatabaseName: "dqrun"
+    DatabaseId: "c9ql09h4firghvrv49jt"
+    Credentials {
+      basic {
+        username: "admin"
+        password: "password"
+      }
+    }
+    Protocol: HTTP
+    UseSsl: true
+	}
+
+	ClusterMapping {
+    Kind: POSTGRESQL,
+    Name: "postgresql_streaming"
+    DatabaseName: "dqrun"
+    DatabaseId: "c9qb2bjghs8onbncpamk"
+    Credentials {
+      basic {
+        username: "crab"
+        password: "password"
+      }
+    }
+    Protocol: NATIVE
+	}
+  MdbGateway: "https://mdb.api.cloud.yandex.net:443"
+}
+
+DbResolver {
+  YdbMvpEndpoint: "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod"
+}
+
+Dq {
+    DefaultSettings {
+        Name: "EnableComputeActor"
+        Value: "1"
+    }
+
+    DefaultSettings {
+        Name: "ComputeActorType"
+        Value: "async"
+    }
+
+    DefaultSettings {
+        Name: "AnalyzeQuery"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "MaxTasksPerStage"
+        Value: "200"
+    }
+
+    DefaultSettings {
+        Name: "MaxTasksPerOperation"
+        Value: "200"
+    }
+
+    DefaultSettings {
+        Name: "EnableInsert"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "_EnablePrecompute"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "UseAggPhases"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "HashJoinMode"
+        Value: "grace"
+    }
+
+    DefaultSettings {
+        Name: "UseFastPickleTransport"
+        Value: "true"
+    }
+}
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.local.conf b/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.local.conf
new file mode 100644
index 0000000000..046e019b3e
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/gateways.local.conf
@@ -0,0 +1,120 @@
+Generic {
+  Connector {
+    Endpoint {
+      host: "localhost"
+      port: 50051
+    }
+    UseSsl: false
+  }
+
+	ClusterMapping {
+    Kind: CLICKHOUSE,
+    Name: "rtmr_dev00_clickhouse_http"
+    DatabaseName: "dqrun"
+    Endpoint {
+      host: "localhost"
+      port: 8123
+    }
+    Credentials {
+      basic {
+        username: "admin"
+        password: "password"
+      }
+    }
+    UseSsl: false
+    Protocol: HTTP
+	}
+
+  ClusterMapping {
+    Kind: CLICKHOUSE,
+    Name: "rtmr_dev00_clickhouse_native"
+    DatabaseName: "dqrun"
+    Endpoint {
+      host: "localhost"
+      port: 9000
+    }
+    Credentials {
+      basic {
+        username: "admin"
+        password: "password"
+      }
+    }
+    UseSsl: false
+    Protocol: NATIVE
+	}
+
+	ClusterMapping {
+    Kind: POSTGRESQL,
+    Name: "rtmr_dev00_postgresql"
+    DatabaseName: "dqrun"
+    Endpoint {
+      host: "localhost"
+      port: 5432
+    }
+    Credentials {
+      basic {
+        username: "admin"
+        password: "password"
+      }
+    }
+    UseSsl: false
+    Protocol: NATIVE
+	}
+
+  DefaultSettings {
+        Name: "DateTimeFormat"
+        Value: "string"
+    }
+}
+
+Dq {
+    DefaultSettings {
+        Name: "EnableComputeActor"
+        Value: "1"
+    }
+
+    DefaultSettings {
+        Name: "ComputeActorType"
+        Value: "async"
+    }
+
+    DefaultSettings {
+        Name: "AnalyzeQuery"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "MaxTasksPerStage"
+        Value: "200"
+    }
+
+    DefaultSettings {
+        Name: "MaxTasksPerOperation"
+        Value: "200"
+    }
+
+    DefaultSettings {
+        Name: "EnableInsert"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "_EnablePrecompute"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "UseAggPhases"
+        Value: "true"
+    }
+
+    DefaultSettings {
+        Name: "HashJoinMode"
+        Value: "grace"
+    }
+
+    DefaultSettings {
+        Name: "UseFastPickleTransport"
+        Value: "true"
+    }
+}
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.different.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.different.yql
new file mode 100644
index 0000000000..5246a8aee2
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.different.yql
@@ -0,0 +1,8 @@
+pragma UseBlocks;
+
+SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2
+	FROM clickhouse_streaming.example_1 as ch
+	JOIN postgresql_streaming.example_2 as pg
+ON ch.id = pg.id
+ORDER BY id;
+
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_1.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_1.yql
new file mode 100644
index 0000000000..2ce5316eb2
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_1.yql
@@ -0,0 +1,8 @@
+pragma UseBlocks;
+
+SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2
+	FROM rtmr_dev00_clickhouse.example_1 as ch
+	JOIN rtmr_dev00_postgresql.example_1 as pg
+ON ch.id = pg.id
+ORDER BY id;
+
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_2.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_2.yql
new file mode 100644
index 0000000000..3684273630
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.cloud.example_2.yql
@@ -0,0 +1,8 @@
+pragma UseBlocks;
+
+SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2
+	FROM clickhouse_streaming.example_2 as ch
+	JOIN postgresql_streaming.example_2 as pg
+ON ch.id = pg.id
+ORDER BY id;
+
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.benchmark_1g.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.benchmark_1g.yql
new file mode 100644
index 0000000000..e7680ef875
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.benchmark_1g.yql
@@ -0,0 +1,4 @@
+SELECT MAX(LENGTH(ch.col) - LENGTH(pg.col))
+	FROM rtmr_dev00_clickhouse.benchmark_1g as ch
+	JOIN rtmr_dev00_postgresql.benchmark_1g as pg
+ON ch.id = pg.id;
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.different.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.different.yql
new file mode 100644
index 0000000000..09be54575c
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.different.yql
@@ -0,0 +1,8 @@
+pragma UseBlocks;
+
+SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2
+	FROM rtmr_dev00_clickhouse.example_1 as ch
+	JOIN rtmr_dev00_postgresql.example_2 as pg
+ON ch.id = pg.id
+ORDER BY id;
+
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.example_1.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.example_1.yql
new file mode 100644
index 0000000000..bc53e7ccc7
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/join.local.example_1.yql
@@ -0,0 +1,8 @@
+pragma UseBlocks;
+
+SELECT ch.id as id, ch.col1 as ch_col1, ch.col2 as ch_col2, pg.col1 as pg_col1, pg.col2 as pg_col2
+	FROM rtmr_dev00_clickhouse_native.example_1 as ch
+	JOIN rtmr_dev00_postgresql.example_1 as pg
+ON ch.id = pg.id
+ORDER BY id;
+
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.cloud.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.cloud.yql
new file mode 100644
index 0000000000..bf4e016605
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.cloud.yql
@@ -0,0 +1,2 @@
+pragma UseBlocks;
+SELECT * FROM postgresql_streaming.example_1;
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.local.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.local.yql
new file mode 100644
index 0000000000..6927470ee6
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.local.yql
@@ -0,0 +1,2 @@
+-- SELECT col_19_bytea FROM rtmr_dev00_postgresql.primitives;
+SELECT MAX(id + LENGTH(col)) FROM rtmr_dev00_postgresql.benchmark_1g;
diff --git a/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.single.yql b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.single.yql
new file mode 100644
index 0000000000..22996a40ab
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/dqrun/pg.single.yql
@@ -0,0 +1,2 @@
+SELECT COL1 FROM rtmr_dev00_postgresql.column_projection;
+
diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-rtmp-dev00.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-rtmp-dev00.txt
new file mode 100644
index 0000000000..a17d84b143
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-rtmp-dev00.txt
@@ -0,0 +1,24 @@
+CREATE OBJECT pg_local_password (TYPE SECRET) WITH (value = qwerty12345);
+
+CREATE EXTERNAL DATA SOURCE pg_local WITH (
+    SOURCE_TYPE="PostgreSQL",
+    LOCATION="rtmr-dev00.search.yandex.net:5432",
+    AUTH_METHOD="BASIC",
+    LOGIN="crab",
+    PASSWORD_SECRET_NAME="pg_local_password",
+    USE_TLS="FALSE",
+    PROTOCOL="NATIVE"
+);
+
+CREATE OBJECT ch_local_password (TYPE SECRET) WITH (value = qwerty12345);
+
+CREATE EXTERNAL DATA SOURCE ch_local WITH (
+    SOURCE_TYPE="ClickHouse",
+    LOCATION="rtmr-dev00.search.yandex.net:9000",
+    AUTH_METHOD="BASIC",
+    LOGIN="crab",
+    PASSWORD_SECRET_NAME="ch_local_password",
+    DATABASE_NAME="dqrun",
+    USE_TLS="FALSE",
+    PROTOCOL="NATIVE"
+);
\ No newline at end of file
diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-streaming.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-streaming.txt
new file mode 100644
index 0000000000..8fbe9e68df
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/kqprun/scheme-streaming.txt
@@ -0,0 +1,25 @@
+-- streaming instances
+
+CREATE OBJECT clickhouse_streaming_password (TYPE SECRET) WITH (value=password);
+CREATE EXTERNAL DATA SOURCE clickhouse_streaming WITH (
+    SOURCE_TYPE="ClickHouse",
+    MDB_CLUSTER_ID="c9ql09h4firghvrv49jt",
+    DATABASE_NAME="dqrun",
+    PROTOCOL="HTTP",
+    AUTH_METHOD="BASIC",
+    LOGIN="admin",
+    PASSWORD_SECRET_NAME="clickhouse_streaming_password",
+    USE_TLS="TRUE"
+);
+
+CREATE OBJECT postgresql_streaming_password (TYPE SECRET) WITH (value=password);
+CREATE EXTERNAL DATA SOURCE postgresql_streaming WITH (
+    SOURCE_TYPE="PostgreSQL",
+    MDB_CLUSTER_ID="c9qb2bjghs8onbncpamk",
+    DATABASE_NAME="dqrun",
+    PROTOCOL="NATIVE",
+    AUTH_METHOD="BASIC",
+    LOGIN="crab",
+    PASSWORD_SECRET_NAME="postgresql_streaming_password",
+    USE_TLS="TRUE"
+);
diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/script.benchmark_1g.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/script.benchmark_1g.txt
new file mode 100644
index 0000000000..2f2ac911da
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/kqprun/script.benchmark_1g.txt
@@ -0,0 +1,5 @@
+SELECT MAX(length(ch.col) + length(pg.col))
+    FROM clickhouse_cloud.benchmark_1g as ch
+    JOIN postgresql_cloud.benchmark_1g as pg
+ON ch.id = pg.id;
+
diff --git a/ydb/library/yql/providers/generic/connector/debug/kqprun/script.example.txt b/ydb/library/yql/providers/generic/connector/debug/kqprun/script.example.txt
new file mode 100644
index 0000000000..7ca381c451
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/kqprun/script.example.txt
@@ -0,0 +1 @@
+SELECT * FROM ch_local.primitives;
diff --git a/ydb/library/yql/providers/generic/connector/debug/postgresql/init/init_db.sh b/ydb/library/yql/providers/generic/connector/debug/postgresql/init/init_db.sh
new file mode 100644
index 0000000000..ca0e4eaaf2
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/postgresql/init/init_db.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DROP TABLE IF EXISTS example_1;
+    CREATE TABLE example_1 (id integer, col1 text, col2 integer);
+    INSERT INTO example_1 VALUES (1, 'pg_example_1_a', 10);
+    INSERT INTO example_1 VALUES (2, 'pg_example_1_b', 20);
+    INSERT INTO example_1 VALUES (3, 'pg_example_1_c', 30);
+    INSERT INTO example_1 VALUES (4, 'pg_example_1_d', 40);
+    INSERT INTO example_1 VALUES (5, 'pg_example_1_e', 50);
+    INSERT INTO example_1 VALUES (6, NULL, 1);
+EOSQL
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DROP TABLE IF EXISTS example_2;
+    CREATE TABLE example_2 (col_01_bool bool, col_02_smallint smallint, col_03_int2 int2, col_04_smallserial smallserial, col_05_serial2 serial2, col_06_integer integer, col_07_int int, col_08_int4 int4, col_09_serial serial, col_10_serial4 serial4, col_11_bigint bigint, col_12_int8 int8, col_13_bigserial bigserial, col_14_serial8 serial8, col_15_real real, col_16_float4 float4, col_17_double_precision double precision, col_18_float8 float8, col_19_bytea bytea, col_20_character character (5), col_21_character_varying character varying (5), col_22_text text, col_23_date date, col_24_timestamp timestamp without time zone);
+    INSERT INTO example_2 (col_01_bool, col_02_smallint, col_03_int2, col_04_smallserial, col_05_serial2, col_06_integer, col_07_int, col_08_int4, col_09_serial, col_10_serial4, col_11_bigint, col_12_int8, col_13_bigserial, col_14_serial8, col_15_real, col_16_float4, col_17_double_precision, col_18_float8, col_19_bytea, col_20_character, col_21_character_varying, col_22_text, col_23_date, col_24_timestamp) VALUES(False, 2, 3, 1, 1, 6, 7, 8, 1, 1, 11, 12, 1, 1, 15.15, 16.16, 17.17, 18.18, 'az', 'az   ', 'az   ', 'az', '2023-08-09', '2023-08-09 13:19:11');
+    INSERT INTO example_2 (col_01_bool, col_02_smallint, col_03_int2, col_04_smallserial, col_05_serial2, col_06_integer, col_07_int, col_08_int4, col_09_serial, col_10_serial4, col_11_bigint, col_12_int8, col_13_bigserial, col_14_serial8, col_15_real, col_16_float4, col_17_double_precision, col_18_float8, col_19_bytea, col_20_character, col_21_character_varying, col_22_text, col_23_date, col_24_timestamp) VALUES(NULL, NULL, NULL, 3, 3, NULL, NULL, NULL, 3, 3, NULL, NULL, 3, 3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+EOSQL
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DROP TABLE IF EXISTS empty;
+    CREATE TABLE empty (id integer, col1 text, col2 integer);
+EOSQL
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DROP TABLE IF EXISTS primitives;
+    CREATE TABLE primitives (
+        col_01_bool bool,
+        col_02_smallint smallint,
+        col_03_int2 int2,
+        col_04_smallserial smallserial,
+        col_05_serial2 serial2,
+        col_06_integer integer,
+        col_07_int int,
+        col_08_int4 int4,
+        col_09_serial serial,
+        col_10_serial4 serial4,
+        col_11_bigint bigint,
+        col_12_int8 int8,
+        col_13_bigserial bigserial,
+        col_14_serial8 serial8,
+        col_15_real real,
+        col_16_float4 float4,
+        col_17_double_precision double precision,
+        col_18_float8 float8,
+        col_19_bytea bytea,
+        col_20_character_n character(20),
+        col_21_character_varying_n character varying(21),
+        col_22_text text,
+        col_23_timestamp timestamp,
+        col_24_date date
+    );
+    INSERT INTO primitives VALUES (
+        true, 2, 3, DEFAULT, DEFAULT, 6, 7, 8, DEFAULT, DEFAULT, 11, 12, DEFAULT, DEFAULT,
+        15.15, 16.16, 17.17, 18.18, 'az', 'az', 'az', 'az',
+        current_timestamp, current_timestamp);
+    INSERT INTO primitives VALUES (
+        false, -2, -3, DEFAULT, DEFAULT, -6, -7, -8, DEFAULT, DEFAULT, -11, -12, DEFAULT, DEFAULT,
+        -15.15, -16.16, -17.17, -18.18, 'буки', 'буки', 'буки', 'буки',
+        current_timestamp, current_timestamp);
+    INSERT INTO primitives VALUES (
+        NULL, NULL, NULL, DEFAULT, DEFAULT, NULL,
+        NULL, NULL, DEFAULT, DEFAULT, NULL, NULL,
+        DEFAULT, DEFAULT, NULL, NULL, NULL, NULL,
+        NULL, NULL, NULL, NULL, NULL, NULL
+        );
+EOSQL
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DROP TABLE IF EXISTS benchmark_1g;
+    CREATE TABLE benchmark_1g (id bigserial, col varchar(1024));
+    INSERT INTO benchmark_1g SELECT generate_series(1,1048576) AS id, REPEAT(md5(random()::text), 32) AS col;
+EOSQL
diff --git a/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/list_schemas.sh b/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/list_schemas.sh
new file mode 100755
index 0000000000..7797eb5d29
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/list_schemas.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -ex
+
+export PGPASSWORD=qwerty12345
+
+SCRIPT="
+select schema_name
+from information_schema.schemata;
+"
+
+sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}"
+
+
+SCRIPT="
+select nspname
+from pg_catalog.pg_namespace;
+"
+
+sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}"
diff --git a/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/show_table_size.sh b/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/show_table_size.sh
new file mode 100755
index 0000000000..c92a8b5b2b
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/postgresql/maintain/show_table_size.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+export PGPASSWORD=qwerty12345
+
+SCRIPT="
+DROP TABLE IF EXISTS benchmark_1g;
+CREATE TABLE benchmark_1g (id bigserial, col varchar(1024));
+INSERT INTO benchmark_1g SELECT generate_series(1,1048576) AS id, REPEAT(md5(random()::text), 32) AS col;
+"
+
+sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}"
+
+SCRIPT="
+SELECT pg_size_pretty(pg_total_relation_size('public.benchmark_1g'));
+"
+
+sudo docker exec -it connector-postgresql psql -U crab -d dqrun -c "${SCRIPT}"
diff --git a/ydb/library/yql/providers/generic/connector/debug/tls/.arcignore b/ydb/library/yql/providers/generic/connector/debug/tls/.arcignore
new file mode 100644
index 0000000000..832b3e16ad
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/tls/.arcignore
@@ -0,0 +1,5 @@
+*.crt
+*.csr
+*.ext
+*.key
+*.pem
diff --git a/ydb/library/yql/providers/generic/connector/debug/tls/generate.sh b/ydb/library/yql/providers/generic/connector/debug/tls/generate.sh
new file mode 100755
index 0000000000..6f613be21d
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/debug/tls/generate.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -ex
+
+# Clean keys for previous run
+rm ca.key ca.crt \
+   server.key server.csr server.crt server.pem server.ext \
+   client.key client.csr client.crt client.pem || true
+
+
+HOSTNAME=$(hostname -f)
+SUBJECT="/C=RU/L=Moscow/O=yq-connector/OU=yandex/CN=${HOSTNAME}/emailAddress=vitalyisaev@yandex-team.ru"
+SUBJECT_ALT_NAME="subjectAltName = DNS:${HOSTNAME},DNS:localhost"
+
+# Generate self signed root CA cert
+openssl req -nodes -x509 -newkey rsa:2048 -keyout ca.key -out ca.crt -subj "${SUBJECT}"
+
+# Server key pair
+openssl req -newkey rsa:2048 -nodes -keyout server.key -subj "${SUBJECT}" -out server.csr
+echo -n "${SUBJECT_ALT_NAME}" > server.ext
+openssl x509 -req -extfile server.ext -days 365 -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt
+cat server.key server.crt > server.pem
+
+# Client key pair
+openssl req -nodes -newkey rsa:2048 -keyout client.key -out client.csr -subj "${SUBJECT}" -addext "${SUBJECT_ALT_NAME}"
+openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAserial ca.srl -out client.crt
+cat client.key client.crt > client.pem