aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVitaly Isaev <vitalyisaev@ydb.tech>2024-04-02 23:43:09 +0300
committerGitHub <noreply@github.com>2024-04-02 23:43:09 +0300
commit9b51a0134cc1093a3f740f484f0f13eb8ad38a77 (patch)
tree7773ce24811766ed31048c9d1a7a7b6aeb61bb40
parent3fe5941bd22fbeb76cc5799ef46455b896e87cc4 (diff)
downloadydb-9b51a0134cc1093a3f740f484f0f13eb8ad38a77.tar.gz
YDB FQ: basic integration tests for YDB as an external data source (#3399)
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py (renamed from ydb/library/yql/providers/generic/connector/tests/test_cases/base.py)13
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py (renamed from ydb/library/yql/providers/generic/connector/tests/test_cases/select_missing_database.py)2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_table.py (renamed from ydb/library/yql/providers/generic/connector/tests/test_cases/select_missing_table.py)2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py (renamed from ydb/library/yql/providers/generic/connector/tests/test_cases/select_positive_common.py)30
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make (renamed from ydb/library/yql/providers/generic/connector/tests/test_cases/ya.make)0
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py6
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py12
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py39
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make3
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py6
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py12
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py45
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make4
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py32
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/ydb/conftest.py4
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml25
-rwxr-xr-xydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/01_basic.sh47
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py50
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make64
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/join/conftest.py13
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/join/scenario.py2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/join/test.py15
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/join/ya.make4
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/clients/ya.make2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/clients/ydb.py10
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/database.py4
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py3
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py (renamed from ydb/library/yql/providers/generic/connector/tests/utils/dqrun.py)29
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py (renamed from ydb/library/yql/providers/generic/connector/tests/utils/kqprun.py)30
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/run/parent.py (renamed from ydb/library/yql/providers/generic/connector/tests/utils/runner.py)14
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/run/result.py15
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/run/runners.py23
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make23
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py8
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py8
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/scenario/ya.make2
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/scenario/ydb.py38
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/schema.py4
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/settings.py39
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/types/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/types/ydb.py79
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/utils/ya.make7
-rw-r--r--ydb/library/yql/providers/generic/connector/tests/ya.make2
49 files changed, 616 insertions, 166 deletions
diff --git a/ydb/library/yql/providers/generic/connector/tests/test_cases/base.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py
index 80fdc5a402a..fb3c11e84f1 100644
--- a/ydb/library/yql/providers/generic/connector/tests/test_cases/base.py
+++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/base.py
@@ -34,7 +34,13 @@ class BaseTestCase:
In general, we cannot use test case name as table name because of special symbols,
so we provide a random table name instead.
'''
- return 't' + hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:8]
+ match self.data_source_kind:
+ case EDataSourceKind.POSTGRESQL:
+ return 't' + hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:8]
+ case EDataSourceKind.CLICKHOUSE:
+ return 't' + hashlib.sha256(str(random.randint(0, 65536)).encode('ascii')).hexdigest()[:8]
+ case EDataSourceKind.YDB:
+ return self.name
@property
def sql_table_name(self) -> str:
@@ -69,5 +75,10 @@ class BaseTestCase:
clickhouse_clusters=[],
postgresql_clusters=[GenericSettings.PostgreSQLCluster(database=self.database.name, schema=None)],
)
+ case EDataSourceKind.YDB:
+ return GenericSettings(
+ date_time_format=EDateTimeFormat.YQL_FORMAT,
+ ydb_clusters=[GenericSettings.YdbCluster(database=self.database.name)],
+ )
case _:
raise Exception(f'invalid data source: {self.data_source_kind}')
diff --git a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_missing_database.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py
index 35a2805f009..ec47a556c42 100644
--- a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_missing_database.py
+++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_database.py
@@ -1,7 +1,7 @@
from typing import List
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol
-from ydb.library.yql.providers.generic.connector.tests.test_cases.base import BaseTestCase
+from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase
TestCase = BaseTestCase
diff --git a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_missing_table.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_table.py
index a9abb5805d9..3a2501e3548 100644
--- a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_missing_table.py
+++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_missing_table.py
@@ -1,7 +1,7 @@
from typing import List
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind, EProtocol
-from ydb.library.yql.providers.generic.connector.tests.test_cases.base import BaseTestCase
+from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase
TestCase = BaseTestCase
diff --git a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_positive_common.py b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py
index 2d5e2e39bd1..fff27b4cb4a 100644
--- a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_positive_common.py
+++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/select_positive_common.py
@@ -9,6 +9,7 @@ from ydb.library.yql.providers.generic.connector.tests.utils.settings import Set
from ydb.library.yql.providers.generic.connector.tests.utils.generate import generate_table_data
import ydb.library.yql.providers.generic.connector.tests.utils.types.clickhouse as clickhouse
import ydb.library.yql.providers.generic.connector.tests.utils.types.postgresql as postgresql
+import ydb.library.yql.providers.generic.connector.tests.utils.types.ydb as Ydb
from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
Schema,
Column,
@@ -18,7 +19,7 @@ from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
SelectWhere,
)
-from ydb.library.yql.providers.generic.connector.tests.test_cases.base import BaseTestCase
+from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase
from ydb.library.yql.providers.generic.connector.tests.utils.settings import GenericSettings
@@ -68,12 +69,12 @@ class Factory:
Column(
name='COL1',
ydb_type=Type.INT32,
- data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4()),
+ data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32()),
),
Column(
name='col2',
ydb_type=Type.INT32,
- data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4()),
+ data_source_type=DataSourceType(ch=clickhouse.Int32(), pg=postgresql.Int4(), ydb=Ydb.Int32()),
),
)
)
@@ -90,6 +91,7 @@ class Factory:
(
EDataSourceKind.CLICKHOUSE,
EDataSourceKind.POSTGRESQL,
+ EDataSourceKind.YDB,
),
),
# SELECT COL1 FROM table
@@ -102,6 +104,7 @@ class Factory:
(
EDataSourceKind.CLICKHOUSE,
# NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
+ EDataSourceKind.YDB,
),
),
# SELECT col1 FROM table
@@ -123,6 +126,7 @@ class Factory:
(
EDataSourceKind.CLICKHOUSE,
EDataSourceKind.POSTGRESQL,
+ EDataSourceKind.YDB,
),
),
# SELECT col2, COL1 FROM table
@@ -135,6 +139,7 @@ class Factory:
(
EDataSourceKind.CLICKHOUSE,
# NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
+ EDataSourceKind.YDB,
),
),
# SELECT col2, col1 FROM table
@@ -157,6 +162,7 @@ class Factory:
(
EDataSourceKind.CLICKHOUSE,
# NOTE: YQ-2264: doesn't work for PostgreSQL because of implicit cast to lowercase (COL1 -> col1)
+ EDataSourceKind.YDB,
),
),
# Select the same column multiple times with different aliases
@@ -176,6 +182,7 @@ class Factory:
(
EDataSourceKind.CLICKHOUSE,
EDataSourceKind.POSTGRESQL,
+ EDataSourceKind.YDB,
),
),
)
@@ -235,7 +242,6 @@ class Factory:
)
data_in = generate_table_data(schema=schema, bytes_soft_limit=table_size)
- print("BIRD", data_in)
# Assuming that request will look something like:
#
@@ -272,14 +278,20 @@ class Factory:
protocols = {
EDataSourceKind.CLICKHOUSE: [EProtocol.NATIVE, EProtocol.HTTP],
EDataSourceKind.POSTGRESQL: [EProtocol.NATIVE],
+ EDataSourceKind.YDB: [EProtocol.NATIVE],
}
- base_test_cases = list(
- itertools.chain(
- self._column_selection(),
- self._large_table(),
+ base_test_cases = None
+
+ if data_source_kind == EDataSourceKind.YDB:
+ base_test_cases = self._column_selection()
+ else:
+ base_test_cases = list(
+ itertools.chain(
+ self._column_selection(),
+ self._large_table(),
+ )
)
- )
test_cases = []
for base_tc in base_test_cases:
diff --git a/ydb/library/yql/providers/generic/connector/tests/test_cases/ya.make b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make
index 49991fd287f..49991fd287f 100644
--- a/ydb/library/yql/providers/generic/connector/tests/test_cases/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/common_test_cases/ya.make
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py
index 16d562445f2..b8f4d5d442f 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/collection.py
@@ -3,9 +3,9 @@ from typing import Sequence, Mapping
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
# test cases
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_database as select_missing_database
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_table as select_missing_table
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
import select_positive
import select_datetime
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py
index 77f1481792a..6f31ef8d80c 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/conftest.py
@@ -3,12 +3,8 @@ import pathlib
import pytest
-import yatest.common as yat
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.dqrun import DqRunner
-from ydb.library.yql.providers.generic.connector.tests.utils.kqprun import KqpRunner
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
from ydb.library.yql.providers.generic.connector.tests.utils.clients.clickhouse import Client, make_client
docker_compose_dir: Final = pathlib.Path("ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse")
@@ -24,11 +20,3 @@ def clickhouse_client(settings) -> Client:
cl = make_client(settings.clickhouse)
yield cl
cl.close()
-
-
-def configure_runner(runner, settings) -> Runner:
- if runner is DqRunner:
- return DqRunner(dqrun_path=yat.build_path("ydb/library/yql/tools/dqrun/dqrun"), settings=settings)
- elif runner is KqpRunner:
- return KqpRunner(kqprun_path=yat.build_path("ydb/tests/tools/kqprun/kqprun"), settings=settings)
- return None
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py
index 18563b855d5..95c390cf250 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_datetime.py
@@ -6,7 +6,7 @@ from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 impo
from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat
from ydb.public.api.protos.ydb_value_pb2 import Type
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
import ydb.library.yql.providers.generic.connector.tests.utils.types.clickhouse as clickhouse
from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
Schema,
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py
index 1fe3500a657..de6af3d7cc5 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/select_positive.py
@@ -19,7 +19,7 @@ from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
makeOptionalYdbTypeFromYdbType,
)
-from ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common import TestCase
+from ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common import TestCase
class Factory:
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py
index 52f33096117..dad7d53bc4e 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/test.py
@@ -2,18 +2,16 @@ import pytest
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
from ydb.library.yql.providers.generic.connector.tests.utils.clients.clickhouse import Client
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-import ydb.library.yql.providers.generic.connector.tests.utils.dqrun as dqrun
-import ydb.library.yql.providers.generic.connector.tests.utils.kqprun as kqprun
+from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner
import ydb.library.yql.providers.generic.connector.tests.utils.scenario.clickhouse as scenario
-from conftest import configure_runner, docker_compose_dir
+from conftest import docker_compose_dir
from collection import Collection
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_database as select_missing_database
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_table as select_missing_table
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
# Global collection of test cases dependent on environment
@@ -21,28 +19,25 @@ tc_collection = Collection(
Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.CLICKHOUSE])
)
-runners = (dqrun.DqRunner, kqprun.KqpRunner)
-runners_ids = ("dqrun", "kqprun")
-
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize("test_case", tc_collection.get('select_positive'), ids=tc_collection.ids('select_positive'))
@pytest.mark.usefixtures("settings")
@pytest.mark.usefixtures("clickhouse_client")
def test_select_positive(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
clickhouse_client: Client,
test_case: select_positive_common.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_positive(
test_name=request.node.name, settings=settings, runner=runner, client=clickhouse_client, test_case=test_case
)
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize(
"test_case", tc_collection.get('select_missing_database'), ids=tc_collection.ids('select_missing_database')
)
@@ -51,11 +46,11 @@ def test_select_positive(
def test_select_missing_database(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
clickhouse_client: Client,
test_case: select_missing_database.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_missing_table(
settings=settings,
runner=runner,
@@ -65,7 +60,7 @@ def test_select_missing_database(
)
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize(
"test_case", tc_collection.get('select_missing_table'), ids=tc_collection.ids('select_missing_table')
)
@@ -74,11 +69,11 @@ def test_select_missing_database(
def test_select_missing_table(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
clickhouse_client: Client,
test_case: select_missing_table.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_missing_table(
test_name=request.node.name,
settings=settings,
@@ -88,18 +83,18 @@ def test_select_missing_table(
)
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize("test_case", tc_collection.get('select_datetime'), ids=tc_collection.ids('select_datetime'))
@pytest.mark.usefixtures("settings")
@pytest.mark.usefixtures("clickhouse_client")
def test_select_datetime(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
clickhouse_client: Client,
test_case: select_positive_common.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_positive(
test_name=request.node.name,
test_case=test_case,
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make
index 3e1760ba092..87d43581681 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make
@@ -50,9 +50,10 @@ PEERDIR(
contrib/python/pytest
ydb/library/yql/providers/generic/connector/api/common
ydb/library/yql/providers/generic/connector/api/service/protos
- ydb/library/yql/providers/generic/connector/tests/test_cases
+ ydb/library/yql/providers/generic/connector/tests/common_test_cases
ydb/library/yql/providers/generic/connector/tests/utils
ydb/library/yql/providers/generic/connector/tests/utils/clients
+ ydb/library/yql/providers/generic/connector/tests/utils/run
ydb/library/yql/providers/generic/connector/tests/utils/scenario
)
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py
index c1ef06c096e..da6a46144b7 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/collection.py
@@ -1,9 +1,9 @@
from typing import Sequence, Mapping
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_database as select_missing_database
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_table as select_missing_table
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
import select_datetime
import select_positive
import select_positive_with_schema
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py
index ee2efeb3da7..e2e36087059 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/conftest.py
@@ -3,13 +3,9 @@ import pathlib
import pytest
-import yatest.common as yat
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.dqrun import DqRunner
-from ydb.library.yql.providers.generic.connector.tests.utils.kqprun import KqpRunner
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
from ydb.library.yql.providers.generic.connector.tests.utils.clients.postgresql import Client
@@ -24,11 +20,3 @@ def settings() -> Settings:
@pytest.fixture
def postgresql_client(settings) -> Client:
return Client(settings.postgresql)
-
-
-def configure_runner(runner, settings) -> Runner:
- if runner is DqRunner:
- return DqRunner(dqrun_path=yat.build_path("ydb/library/yql/tools/dqrun/dqrun"), settings=settings)
- elif runner is KqpRunner:
- return KqpRunner(kqprun_path=yat.build_path("ydb/tests/tools/kqprun/kqprun"), settings=settings)
- return None
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py
index 1a68c6ac375..7c396bd5013 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_datetime.py
@@ -6,7 +6,7 @@ from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 impo
from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat
from ydb.public.api.protos.ydb_value_pb2 import Type
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
import ydb.library.yql.providers.generic.connector.tests.utils.types.postgresql as postgresql
from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
Schema,
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py
index 30115a7b0d3..ac605f6576f 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive.py
@@ -16,7 +16,7 @@ from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
makeOptionalYdbTypeFromTypeID,
)
-from ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common import TestCase
+from ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common import TestCase
class Factory:
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py
index 0cc4bf2f8ee..b2cb9bf457c 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/select_positive_with_schema.py
@@ -17,7 +17,7 @@ from ydb.library.yql.providers.generic.connector.tests.utils.schema import (
DataSourceType,
SelectWhat,
)
-from ydb.library.yql.providers.generic.connector.tests.test_cases.base import BaseTestCase
+from ydb.library.yql.providers.generic.connector.tests.common_test_cases.base import BaseTestCase
@dataclass
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py
index 9a573ef12bd..4483d6201d2 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/test.py
@@ -2,18 +2,16 @@ import pytest
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
-import ydb.library.yql.providers.generic.connector.tests.utils.dqrun as dqrun
-import ydb.library.yql.providers.generic.connector.tests.utils.kqprun as kqprun
+from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner
from ydb.library.yql.providers.generic.connector.tests.utils.clients.postgresql import Client
import ydb.library.yql.providers.generic.connector.tests.utils.scenario.postgresql as scenario
-from conftest import configure_runner, docker_compose_dir
+from conftest import docker_compose_dir
from collection import Collection
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_database as select_missing_database
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_table as select_missing_table
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
# Global collection of test cases dependent on environment
@@ -21,22 +19,19 @@ tc_collection = Collection(
Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.POSTGRESQL])
)
-runners = (dqrun.DqRunner, kqprun.KqpRunner)
-runners_ids = ("dqrun", "kqprun")
-
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize("test_case", tc_collection.get('select_positive'), ids=tc_collection.ids('select_positive'))
@pytest.mark.usefixtures("settings")
@pytest.mark.usefixtures("postgresql_client")
def test_select_positive(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
postgresql_client: Client,
test_case: select_positive_common.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_positive(
settings=settings,
runner=runner,
@@ -46,7 +41,7 @@ def test_select_positive(
)
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize(
"test_case", tc_collection.get('select_missing_database'), ids=tc_collection.ids('select_missing_database')
)
@@ -55,11 +50,11 @@ def test_select_positive(
def test_select_missing_database(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
postgresql_client: Client,
test_case: select_missing_database.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_missing_table(
settings=settings,
runner=runner,
@@ -69,7 +64,7 @@ def test_select_missing_database(
)
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize(
"test_case", tc_collection.get('select_missing_table'), ids=tc_collection.ids('select_missing_table')
)
@@ -78,11 +73,11 @@ def test_select_missing_database(
def test_select_missing_table(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
postgresql_client: Client,
test_case: select_missing_table.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_missing_table(
test_name=request.node.name,
settings=settings,
@@ -92,18 +87,18 @@ def test_select_missing_table(
)
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize("test_case", tc_collection.get('select_datetime'), ids=tc_collection.ids('select_datetime'))
@pytest.mark.usefixtures("settings")
@pytest.mark.usefixtures("postgresql_client")
def test_select_datetime(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
postgresql_client: Client,
test_case: select_positive_common.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_positive(
settings=settings,
runner=runner,
@@ -113,7 +108,7 @@ def test_select_datetime(
)
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize(
"test_case",
tc_collection.get('select_positive_with_schema'),
@@ -124,11 +119,11 @@ def test_select_datetime(
def test_select_positive_with_schema(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
postgresql_client: Client,
test_case: select_positive_common.TestCase,
):
- runner = configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.select_pg_schema(
settings=settings,
runner=runner,
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make
index 9da05844c71..9c622fcf3b7 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make
@@ -50,9 +50,9 @@ TEST_SRCS(
PEERDIR(
contrib/python/pytest
ydb/library/yql/providers/generic/connector/api/common
- ydb/library/yql/providers/generic/connector/tests/test_cases
+ ydb/library/yql/providers/generic/connector/tests/common_test_cases
ydb/library/yql/providers/generic/connector/tests/utils
- ydb/library/yql/providers/generic/connector/tests/utils/types
+ ydb/library/yql/providers/generic/connector/tests/utils/run
ydb/library/yql/providers/generic/connector/tests/utils/clients
ydb/library/yql/providers/generic/connector/tests/utils/scenario
)
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/ya.make
index 787988204ef..3f0f2c8f011 100644
--- a/ydb/library/yql/providers/generic/connector/tests/datasource/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ya.make
@@ -1,4 +1,5 @@
RECURSE_FOR_TESTS(
clickhouse
postgresql
+ ydb
)
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py
new file mode 100644
index 00000000000..3c747d5fc9e
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py
@@ -0,0 +1,32 @@
+from typing import Sequence, Mapping
+
+from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
+
+# import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+# import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
+
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+
+class Collection(object):
+ _test_cases: Mapping[str, Sequence]
+
+ def __init__(self, ss: Settings):
+ self._test_cases = {
+ # 'select_missing_database': select_missing_database.Factory().make_test_cases(EDataSourceKind.YDB),
+ # 'select_missing_table': select_missing_table.Factory().make_test_cases(EDataSourceKind.YDB),
+ 'select_positive': select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.YDB),
+ }
+
+ def get(self, key: str) -> Sequence:
+ if key not in self._test_cases:
+ raise ValueError(f'no such test: {key}')
+
+ return self._test_cases[key]
+
+ def ids(self, key: str) -> Sequence[str]:
+ if key not in self._test_cases:
+ raise ValueError(f'no such test: {key}')
+
+ return map(lambda tc: tc.name, self._test_cases[key])
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/conftest.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/conftest.py
new file mode 100644
index 00000000000..5e12b074140
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/conftest.py
@@ -0,0 +1,4 @@
+from typing import Final
+import pathlib
+
+docker_compose_dir: Final = pathlib.Path("ydb/library/yql/providers/generic/connector/tests/datasource/ydb")
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml
new file mode 100644
index 00000000000..93a54ce2ffc
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml
@@ -0,0 +1,25 @@
+version: '3.4'
+services:
+ ydb:
+ image: cr.yandex/yc/yandex-docker-local-ydb:23.3.17@sha256:bf9001c849cc6c4c9b56f32f5440a6e8390c4e841937c9f9caf929fd70a689c8
+ container_name: fq-tests-ydb-ydb
+ hostname: fq-tests-ydb-ydb
+ environment:
+ YDB_DEFAULT_LOG_LEVEL: INFO
+ POSTGRES_USER: user
+ POSTGRES_PASSWORD: password
+ volumes:
+ - ./init/init_ydb:/init_ydb
+ - ./init/01_basic.sh:/01_basic.sh
+
+ fq-connector-go:
+ image: ghcr.io/ydb-platform/fq-connector-go:v0.2.12@sha256:dd2483ba061e25e8ee645bcc64cae8b8a0a93dba6772eb4b8ab0a0aab4b8dd48
+ container_name: fq-tests-ydb-fq-connector-go
+ volumes:
+ - ../../fq-connector-go/:/opt/ydb/cfg/
+ ports:
+ - 2130
+ command: >
+ sh -c "
+ echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts;
+ /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/01_basic.sh b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/01_basic.sh
new file mode 100755
index 00000000000..28daf80535d
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/01_basic.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+/ydb -p tests-ydb-client yql -s '
+ CREATE TABLE column_selection_A_b_C_d_E_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1));
+ COMMIT;
+ INSERT INTO column_selection_A_b_C_d_E_NATIVE (COL1, col2) VALUES
+ (1, 2),
+ (10, 20);
+ COMMIT;
+
+ CREATE TABLE column_selection_COL1_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1));
+ COMMIT;
+ INSERT INTO column_selection_COL1_NATIVE (COL1, col2) VALUES
+ (1, 2),
+ (10, 20);
+ COMMIT;
+
+ CREATE TABLE column_selection_asterisk_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1));
+ COMMIT;
+ INSERT INTO column_selection_asterisk_NATIVE (COL1, col2) VALUES
+ (1, 2),
+ (10, 20);
+ COMMIT;
+
+ CREATE TABLE column_selection_col2_COL1_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1));
+ COMMIT;
+ INSERT INTO column_selection_col2_COL1_NATIVE (COL1, col2) VALUES
+ (1, 2),
+ (10, 20);
+ COMMIT;
+
+ CREATE TABLE column_selection_col2_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1));
+ COMMIT;
+ INSERT INTO column_selection_col2_NATIVE (COL1, col2) VALUES
+ (1, 2),
+ (10, 20);
+ COMMIT;
+
+ CREATE TABLE column_selection_col3_NATIVE (COL1 Int32, col2 Int32, PRIMARY KEY (COL1));
+ COMMIT;
+ INSERT INTO column_selection_col3_NATIVE (COL1, col2) VALUES
+ (1, 2),
+ (10, 20);
+ COMMIT;
+ '
+
+echo $(date +"%T.%6N") "SUCCESS"
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py
new file mode 100644
index 00000000000..288782268cd
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/test.py
@@ -0,0 +1,50 @@
+import pytest
+import time
+
+from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner
+import ydb.library.yql.providers.generic.connector.tests.utils.scenario.ydb as scenario
+
+from conftest import docker_compose_dir
+from collection import Collection
+
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
+
+
+class OneTimeWaiter:
+ __launched: bool = False
+
+ def wait(self):
+ if self.__launched:
+ return
+
+ # This should be enough for tables to initialize
+ time.sleep(3)
+ self.__launched = True
+
+
+one_time_waiter = OneTimeWaiter()
+
+settings = Settings.from_env(docker_compose_dir=docker_compose_dir, data_source_kinds=[EDataSourceKind.YDB])
+tc_collection = Collection(settings)
+
+
+@pytest.mark.parametrize("runner_type", runner_types)
+@pytest.mark.parametrize("test_case", tc_collection.get('select_positive'), ids=tc_collection.ids('select_positive'))
+def test_select_positive(
+ request: pytest.FixtureRequest,
+ runner_type: str,
+ test_case: select_positive_common.TestCase,
+):
+ # Let YDB container initialize tables
+ one_time_waiter.wait()
+
+ runner = configure_runner(runner_type=runner_type, settings=settings)
+ scenario.select_positive(
+ settings=settings,
+ runner=runner,
+ test_case=test_case,
+ test_name=request.node.name,
+ )
diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make
new file mode 100644
index 00000000000..e79c76b3f4a
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make
@@ -0,0 +1,64 @@
+PY3TEST()
+
+STYLE_PYTHON()
+NO_CHECK_IMPORTS()
+
+DATA(arcadia/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml)
+DATA(arcadia/ydb/library/yql/providers/generic/connector/tests/fq-connector-go)
+ENV(COMPOSE_PROJECT_NAME=ydb)
+
+IF (AUTOCHECK)
+ # Split tests to chunks only when they're running on different machines with distbuild,
+ # otherwise this directive will slow down local test execution.
+ # Look through https://st.yandex-team.ru/DEVTOOLSSUPPORT-39642 for more information.
+ FORK_SUBTESTS()
+
+ # TAG and REQUIREMENTS are copied from: https://docs.yandex-team.ru/devtools/test/environment#docker-compose
+ TAG(
+ ya:external
+ ya:force_sandbox
+ ya:fat
+ )
+
+ REQUIREMENTS(
+ container:4467981730
+ cpu:all
+ dns:dns64
+ )
+ENDIF()
+
+INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc)
+
+# Including of docker_compose/recipe.inc automatically converts these tests into LARGE,
+# which makes it impossible to run them during precommit checks on Github CI.
+# Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928.
+
+IF (OPENSOURCE)
+ SIZE(MEDIUM)
+ SET(TEST_TAGS_VALUE)
+ SET(TEST_REQUIREMENTS_VALUE)
+ENDIF()
+
+TEST_SRCS(
+ collection.py
+ conftest.py
+ test.py
+)
+
+PEERDIR(
+ contrib/python/pytest
+ ydb/library/yql/providers/generic/connector/api/common
+ ydb/library/yql/providers/generic/connector/tests/common_test_cases
+ ydb/library/yql/providers/generic/connector/tests/utils
+ ydb/library/yql/providers/generic/connector/tests/utils/run
+ ydb/library/yql/providers/generic/connector/tests/utils/clients
+ ydb/library/yql/providers/generic/connector/tests/utils/scenario
+)
+
+DEPENDS(
+ ydb/library/yql/tools/dqrun
+ ydb/tests/tools/kqprun
+ library/recipes/docker_compose/bin
+)
+
+END()
diff --git a/ydb/library/yql/providers/generic/connector/tests/join/conftest.py b/ydb/library/yql/providers/generic/connector/tests/join/conftest.py
index 241f43be6b3..19caea7afe9 100644
--- a/ydb/library/yql/providers/generic/connector/tests/join/conftest.py
+++ b/ydb/library/yql/providers/generic/connector/tests/join/conftest.py
@@ -4,8 +4,6 @@ import pathlib
import pytest
-import yatest.common as yat
-
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
from ydb.library.yql.providers.generic.connector.tests.utils.clients.clickhouse import (
make_client as make_clickhouse_client,
@@ -13,9 +11,6 @@ from ydb.library.yql.providers.generic.connector.tests.utils.clients.clickhouse
)
from ydb.library.yql.providers.generic.connector.tests.utils.clients.postgresql import Client as PostgreSQLClient
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.dqrun import DqRunner
-from ydb.library.yql.providers.generic.connector.tests.utils.kqprun import KqpRunner
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
docker_compose_dir: Final = pathlib.Path("ydb/library/yql/providers/generic/connector/tests/join")
@@ -41,11 +36,3 @@ def clients(settings):
ClickHouse=make_clickhouse_client(settings=settings.clickhouse),
PostgreSQL=PostgreSQLClient(settings=settings.postgresql),
)
-
-
-def configure_runner(runner, settings) -> Runner:
- if runner is DqRunner:
- return DqRunner(dqrun_path=yat.build_path("ydb/library/yql/tools/dqrun/dqrun"), settings=settings)
- elif runner is KqpRunner:
- return KqpRunner(kqprun_path=yat.build_path("ydb/tests/tools/kqprun/kqprun"), settings=settings)
- return None
diff --git a/ydb/library/yql/providers/generic/connector/tests/join/scenario.py b/ydb/library/yql/providers/generic/connector/tests/join/scenario.py
index 5d02e416f56..3eff1950d8c 100644
--- a/ydb/library/yql/providers/generic/connector/tests/join/scenario.py
+++ b/ydb/library/yql/providers/generic/connector/tests/join/scenario.py
@@ -2,7 +2,7 @@ from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 impo
from ydb.library.yql.providers.generic.connector.tests.utils.comparator import data_outs_equal
from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
+from ydb.library.yql.providers.generic.connector.tests.utils.run.parent import Runner
from ydb.library.yql.providers.generic.connector.tests.utils.clients.clickhouse import Client as ClickHouseClient
import ydb.library.yql.providers.generic.connector.tests.utils.scenario.clickhouse as clickhouse_scenario
diff --git a/ydb/library/yql/providers/generic/connector/tests/join/test.py b/ydb/library/yql/providers/generic/connector/tests/join/test.py
index b1226258cbc..8af6b76f5a2 100644
--- a/ydb/library/yql/providers/generic/connector/tests/join/test.py
+++ b/ydb/library/yql/providers/generic/connector/tests/join/test.py
@@ -2,11 +2,9 @@ import pytest
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
-import ydb.library.yql.providers.generic.connector.tests.utils.dqrun as dqrun
-import ydb.library.yql.providers.generic.connector.tests.utils.kqprun as kqprun
+from ydb.library.yql.providers.generic.connector.tests.utils.run.runners import runner_types, configure_runner
-import conftest # import configure_runner, docker_compose_dir, Clients, clients
+import conftest
import scenario
from collection import Collection
from test_case import TestCase
@@ -19,22 +17,19 @@ tc_collection = Collection(
)
)
-runners = (dqrun.DqRunner, kqprun.KqpRunner)
-runners_ids = ("dqrun", "kqprun")
-
-@pytest.mark.parametrize("runner_type", runners, ids=runners_ids)
+@pytest.mark.parametrize("runner_type", runner_types)
@pytest.mark.parametrize("test_case", tc_collection.get('join'), ids=tc_collection.ids('join'))
@pytest.mark.usefixtures("settings")
@pytest.mark.usefixtures("clients")
def test_join(
request: pytest.FixtureRequest,
settings: Settings,
- runner_type: Runner,
+ runner_type: str,
clients: conftest.Clients,
test_case: TestCase,
):
- runner = conftest.configure_runner(runner=runner_type, settings=settings)
+ runner = configure_runner(runner_type=runner_type, settings=settings)
scenario.join(
test_name=request.node.name,
clickhouse_client=clients.ClickHouse,
diff --git a/ydb/library/yql/providers/generic/connector/tests/join/ya.make b/ydb/library/yql/providers/generic/connector/tests/join/ya.make
index 5870fb5bef8..74b7e810afb 100644
--- a/ydb/library/yql/providers/generic/connector/tests/join/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/join/ya.make
@@ -50,11 +50,11 @@ PEERDIR(
contrib/python/pytest
ydb/library/yql/providers/generic/connector/api/common
ydb/library/yql/providers/generic/connector/api/service/protos
- ydb/library/yql/providers/generic/connector/tests/test_cases
+ ydb/library/yql/providers/generic/connector/tests/common_test_cases
ydb/library/yql/providers/generic/connector/tests/utils
ydb/library/yql/providers/generic/connector/tests/utils/clients
+ ydb/library/yql/providers/generic/connector/tests/utils/run
ydb/library/yql/providers/generic/connector/tests/utils/scenario
- ydb/library/yql/providers/generic/connector/tests/utils/types
)
DEPENDS(
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/clients/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/clients/ya.make
index 6b9af263fcf..c0845060e51 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/clients/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/clients/ya.make
@@ -3,11 +3,13 @@ PY3_LIBRARY()
PY_SRCS(
clickhouse.py
postgresql.py
+ ydb.py
)
PEERDIR(
contrib/python/clickhouse-connect
contrib/python/pg8000
+ ydb/public/sdk/python
ydb/library/yql/providers/generic/connector/tests/utils
)
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/clients/ydb.py b/ydb/library/yql/providers/generic/connector/tests/utils/clients/ydb.py
new file mode 100644
index 00000000000..f4e2345789e
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/clients/ydb.py
@@ -0,0 +1,10 @@
+import ydb
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+
+def make_client(s: Settings.Ydb) -> ydb.Driver:
+ endpoint = f"grpc://{s.host_external}:{s.port_external}"
+
+ driver = ydb.Driver(endpoint=endpoint, database=s.dbname, credentials=ydb.AnonymousCredentials())
+ driver.wait(timeout=5)
+ return driver
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/database.py b/ydb/library/yql/providers/generic/connector/tests/utils/database.py
index c691aed4275..5b9c9cbe930 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/database.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/database.py
@@ -17,6 +17,10 @@ class Database:
self.name = name[:63].lower()
case EDataSourceKind.CLICKHOUSE:
self.name = name[:255]
+ case EDataSourceKind.YDB:
+ # We use a different way of initialization when working with YDB.
+ # There is only one preinstalled database called
+ self.name = "local"
case _:
raise Exception(f'invalid data source: {self.kind}')
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py b/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py
index 014bf2f8032..98ca5788cf7 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/docker_compose.py
@@ -93,3 +93,6 @@ class EndpointDeterminer:
return out
except subprocess.CalledProcessError as e:
raise RuntimeError(f"docker-compose error: {e.output} (code {e.returncode})")
+
+ def get_container_name(self, service_name: str) -> str:
+ return self.docker_compose_yml_data['services'][service_name]['container_name']
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/dqrun.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py
index f3ee81780d8..5ffc0140e9c 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/dqrun.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/dqrun.py
@@ -10,16 +10,18 @@ from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 impo
from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat
import ydb.library.yql.providers.generic.connector.tests.utils.artifacts as artifacts
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Result, Runner
from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger
from ydb.library.yql.providers.generic.connector.tests.utils.schema import Schema
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings, GenericSettings
+from ydb.library.yql.providers.generic.connector.tests.utils.run.parent import Runner
+from ydb.library.yql.providers.generic.connector.tests.utils.run.result import Result
+
LOGGER = make_logger(__name__)
class GatewaysConfRenderer:
- template_: Final = '''
+ _template: Final = '''
Generic {
Connector {
Endpoint {
@@ -99,6 +101,26 @@ Generic {
}}
{% endfor %}
+{% for cluster in generic_settings.ydb_clusters %}
+ ClusterMapping {
+ Kind: YDB
+ Name: "{{settings.ydb.cluster_name}}"
+ DatabaseName: "{{cluster.database}}"
+ Credentials {
+ basic {
+ username: "{{settings.ydb.username}}"
+ password: "{{settings.ydb.password}}"
+ }
+ }
+ Endpoint {
+ host: "{{settings.ydb.host_internal}}"
+ port: {{settings.ydb.port_internal}}
+ }
+ UseSsl: false
+ Protocol: NATIVE
+ }
+{% endfor %}
+
DefaultSettings {
Name: "DateTimeFormat"
{% if generic_settings.date_time_format == EDateTimeFormat.STRING_FORMAT %}
@@ -107,7 +129,6 @@ Generic {
Value: "YQL"
{% endif %}
}
-
}
Dq {
@@ -165,7 +186,7 @@ Dq {
def __init__(self):
self.template = jinja2.Environment(loader=jinja2.BaseLoader, undefined=jinja2.DebugUndefined).from_string(
- self.template_
+ self._template
)
self.template.globals['EProtocol'] = EProtocol
self.template.globals['EDateTimeFormat'] = EDateTimeFormat
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/kqprun.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py
index 603f8fd96c3..9433d4db229 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/kqprun.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/kqprun.py
@@ -1,20 +1,21 @@
from pathlib import Path
-import subprocess
from typing import Final
+import json
+import subprocess
import jinja2
-import json
-
from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EProtocol
from ydb.library.yql.providers.generic.connector.api.service.protos.connector_pb2 import EDateTimeFormat
import ydb.library.yql.providers.generic.connector.tests.utils.artifacts as artifacts
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Result, Runner
from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger
from ydb.library.yql.providers.generic.connector.tests.utils.schema import Schema
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings, GenericSettings
+from ydb.library.yql.providers.generic.connector.tests.utils.run.parent import Runner
+from ydb.library.yql.providers.generic.connector.tests.utils.run.result import Result
+
LOGGER = make_logger(__name__)
@@ -32,8 +33,10 @@ CREATE EXTERNAL DATA SOURCE {{data_source}} WITH (
AUTH_METHOD="BASIC",
LOGIN="{{login}}",
PASSWORD_SECRET_NAME="{{data_source}}_local_password",
- USE_TLS="FALSE",
- PROTOCOL="{{protocol}}"
+ {% if protocol %}
+ PROTOCOL="{{protocol}}",
+ {% endif %}
+ USE_TLS="FALSE"
{% if kind == POSTGRESQL and schema %}
,SCHEMA="{{schema}}"
@@ -44,6 +47,7 @@ CREATE EXTERNAL DATA SOURCE {{data_source}} WITH (
{% set CLICKHOUSE = 'ClickHouse' %}
{% set POSTGRESQL = 'PostgreSQL' %}
+{% set YDB = 'Ydb' %}
{% set NATIVE = 'NATIVE' %}
{% set HTTP = 'HTTP' %}
@@ -85,6 +89,20 @@ CREATE EXTERNAL DATA SOURCE {{data_source}} WITH (
}}
{% endfor %}
+{% for cluster in generic_settings.ydb_clusters %}
+{{ create_data_source(
+ YDB,
+ settings.ydb.cluster_name,
+ settings.ydb.host_internal,
+ settings.ydb.port_internal,
+ settings.ydb.username,
+ settings.ydb.password,
+ NONE,
+ cluster.database,
+ NONE)
+}}
+{% endfor %}
+
'''
def __init__(self):
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/runner.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/parent.py
index d5d4b12e100..d297efd01db 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/runner.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/parent.py
@@ -1,20 +1,8 @@
from abc import ABC, abstractmethod
-from dataclasses import dataclass
from pathlib import Path
-from typing import List, Optional
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings, GenericSettings
-from ydb.library.yql.providers.generic.connector.tests.utils.schema import Schema, YsonList
-
-
-@dataclass
-class Result:
- data_out: Optional[YsonList]
- data_out_with_types: Optional[List]
- schema: Optional[Schema]
- stdout: str
- stderr: str
- returncode: int
+from ydb.library.yql.providers.generic.connector.tests.utils.run.result import Result
class Runner(ABC):
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/result.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/result.py
new file mode 100644
index 00000000000..45078d16bd0
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/result.py
@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+from typing import Optional, List
+
+from yt import yson
+from ydb.library.yql.providers.generic.connector.tests.utils.schema import Schema
+
+
+@dataclass
+class Result:
+ data_out: Optional[yson.yson_types.YsonList]
+ data_out_with_types: Optional[List]
+ schema: Optional[Schema]
+ stdout: str
+ stderr: str
+ returncode: int
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/runners.py b/ydb/library/yql/providers/generic/connector/tests/utils/run/runners.py
new file mode 100644
index 00000000000..435502c7d55
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/runners.py
@@ -0,0 +1,23 @@
+from typing import Final
+
+import yatest.common as yat
+
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+
+from ydb.library.yql.providers.generic.connector.tests.utils.run.parent import Runner
+from ydb.library.yql.providers.generic.connector.tests.utils.run.dqrun import DqRunner
+from ydb.library.yql.providers.generic.connector.tests.utils.run.kqprun import KqpRunner
+
+# used in every test.py
+runner_types: Final = ("dqrun", "kqprun")
+
+
+# used in every test.py
+def configure_runner(runner_type: str, settings: Settings) -> Runner:
+ match runner_type:
+ case "dqrun":
+ return DqRunner(dqrun_path=yat.build_path("ydb/library/yql/tools/dqrun/dqrun"), settings=settings)
+ case "kqprun":
+ return KqpRunner(kqprun_path=yat.build_path("ydb/tests/tools/kqprun/kqprun"), settings=settings)
+ case _:
+ raise ValueError(runner_type)
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make
new file mode 100644
index 00000000000..8173db5b86d
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/run/ya.make
@@ -0,0 +1,23 @@
+PY3_LIBRARY()
+
+STYLE_PYTHON()
+
+PY_SRCS(
+ dqrun.py
+ kqprun.py
+ parent.py
+ result.py
+ runners.py
+)
+
+PEERDIR(
+ contrib/python/Jinja2
+ contrib/python/PyYAML
+ ydb/library/yql/providers/generic/connector/api/common
+ ydb/library/yql/providers/generic/connector/api/service/protos
+ ydb/library/yql/providers/generic/connector/tests/utils
+ ydb/public/api/protos
+ yt/python/yt/yson
+)
+
+END()
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py
index 7dae5b95232..2429556ffaf 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/clickhouse.py
@@ -8,13 +8,13 @@ from ydb.library.yql.providers.generic.connector.tests.utils.database import Dat
from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger, debug_with_limit
from ydb.library.yql.providers.generic.connector.tests.utils.schema import Schema
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
+from ydb.library.yql.providers.generic.connector.tests.utils.run.parent import Runner
from ydb.library.yql.providers.generic.connector.tests.utils.sql import format_values_for_bulk_sql_insert
from ydb.library.yql.providers.generic.connector.tests.utils.clients.clickhouse import Client
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_database as tc_select_missing_database
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_table as tc_select_missing_table
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as tc_select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as tc_select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as tc_select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as tc_select_positive_common
LOGGER = make_logger(__name__)
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py
index bef4fd47673..babcc37abd1 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/postgresql.py
@@ -8,13 +8,13 @@ from ydb.library.yql.providers.generic.connector.tests.utils.database import Dat
from ydb.library.yql.providers.generic.connector.tests.utils.log import make_logger, debug_with_limit
from ydb.library.yql.providers.generic.connector.tests.utils.schema import Schema
from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
-from ydb.library.yql.providers.generic.connector.tests.utils.runner import Runner
+from ydb.library.yql.providers.generic.connector.tests.utils.run.parent import Runner
from ydb.library.yql.providers.generic.connector.tests.utils.sql import format_values_for_bulk_sql_insert
from ydb.library.yql.providers.generic.connector.tests.utils.clients.postgresql import Client
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_positive_common as tc_select_positive_common
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_database as tc_select_missing_database
-import ydb.library.yql.providers.generic.connector.tests.test_cases.select_missing_table as tc_select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as tc_select_positive_common
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as tc_select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as tc_select_missing_table
LOGGER = make_logger(__name__)
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/ya.make
index a84afe859d0..d4067949140 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/ya.make
@@ -3,6 +3,7 @@ PY3_LIBRARY()
PY_SRCS(
clickhouse.py
postgresql.py
+ ydb.py
)
PEERDIR(
@@ -10,6 +11,7 @@ PEERDIR(
contrib/python/pg8000
ydb/library/yql/providers/generic/connector/tests/utils
ydb/library/yql/providers/generic/connector/tests/utils/clients
+ ydb/library/yql/providers/generic/connector/tests/utils/run
)
END()
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/scenario/ydb.py b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/ydb.py
new file mode 100644
index 00000000000..67f96b15b87
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/scenario/ydb.py
@@ -0,0 +1,38 @@
+from ydb.library.yql.providers.generic.connector.tests.utils.comparator import data_outs_equal
+from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
+from ydb.library.yql.providers.generic.connector.tests.utils.run.parent import Runner
+
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as tc_select_positive_common
+
+
+def select_positive(
+ test_name: str,
+ test_case: tc_select_positive_common.TestCase,
+ settings: Settings,
+ runner: Runner,
+):
+ # read data
+ where_statement = ""
+ if test_case.select_where is not None:
+ where_statement = "WHERE " + test_case.select_where.render(
+ cluster_name=settings.ydb.cluster_name,
+ table_name=test_case.qualified_table_name,
+ )
+ yql_script = f"""
+ {test_case.pragmas_sql_string}
+ SELECT {test_case.select_what.yql_select_names}
+ FROM {settings.ydb.cluster_name}.{test_case.qualified_table_name}
+ {where_statement}
+ """
+ result = runner.run(
+ test_name=test_name,
+ script=yql_script,
+ generic_settings=test_case.generic_settings,
+ )
+
+ assert result.returncode == 0, result.stderr
+
+ assert data_outs_equal(test_case.data_out, result.data_out_with_types), (
+ test_case.data_out,
+ result.data_out_with_types,
+ )
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/schema.py b/ydb/library/yql/providers/generic/connector/tests/utils/schema.py
index fee487bf0f6..e0ba178cae1 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/schema.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/schema.py
@@ -10,6 +10,7 @@ from ydb.public.api.protos.ydb_value_pb2 import Type, OptionalType
import ydb.library.yql.providers.generic.connector.tests.utils.types.clickhouse as clickhouse
import ydb.library.yql.providers.generic.connector.tests.utils.types.postgresql as postgresql
+import ydb.library.yql.providers.generic.connector.tests.utils.types.ydb as Ydb
YsonList: TypeAlias = yson.yson_types.YsonList
@@ -18,6 +19,7 @@ YsonList: TypeAlias = yson.yson_types.YsonList
class DataSourceType:
ch: clickhouse.Type = None
pg: postgresql.Type = None
+ ydb: Ydb.Type = None
def pick(self, kind: EDataSourceKind.ValueType) -> str:
target = None
@@ -26,6 +28,8 @@ class DataSourceType:
target = self.ch
case EDataSourceKind.POSTGRESQL:
target = self.pg
+ case EDataSourceKind.YDB:
+ target = self.ydb
case _:
raise Exception(f'invalid data source: {kind}')
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/settings.py b/ydb/library/yql/providers/generic/connector/tests/utils/settings.py
index 9b48c4af42f..51971998665 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/settings.py
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/settings.py
@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from typing import Optional, Sequence
import pathlib
@@ -40,7 +40,7 @@ class Settings:
dbname: str
cluster_name: str
username: str
- password: Optional[str]
+ password: Optional[str] # TODO: why optional?
host_external: str
host_internal: str
port_external: int
@@ -48,6 +48,17 @@ class Settings:
postgresql: PostgreSQL
+ @dataclass
+ class Ydb:
+ dbname: str
+ cluster_name: str
+ username: str
+ password: str
+ host_internal: str
+ port_internal: int
+
+ ydb: Ydb
+
@classmethod
def from_env(cls, docker_compose_dir: pathlib.Path, data_source_kinds: Sequence[EDataSourceKind]) -> 'Settings':
docker_compose_file_relative_path = str(docker_compose_dir / 'docker-compose.yml')
@@ -88,11 +99,18 @@ class Settings:
username='user',
password='password',
)
+ case EDataSourceKind.YDB:
+ data_sources[data_source_kind] = cls.Ydb(
+ cluster_name='ydb_integration_test',
+ host_internal=endpoint_determiner.get_container_name('ydb'),
+ port_internal=2136,
+ dbname="local",
+ username='user',
+ password='password',
+ )
case _:
raise Exception(f'invalid data source: {data_source_kind}')
- print("CRAB", data_sources)
-
return cls(
connector=cls.Connector(
grpc_host='localhost',
@@ -102,6 +120,7 @@ class Settings:
),
clickhouse=data_sources.get(EDataSourceKind.CLICKHOUSE),
postgresql=data_sources.get(EDataSourceKind.POSTGRESQL),
+ ydb=data_sources.get(EDataSourceKind.YDB),
)
def get_cluster_name(self, data_source_kind: EDataSourceKind) -> str:
@@ -116,6 +135,8 @@ class Settings:
@dataclass
class GenericSettings:
+ date_time_format: EDateTimeFormat
+
@dataclass
class ClickHouseCluster:
def __hash__(self) -> int:
@@ -124,7 +145,7 @@ class GenericSettings:
database: str
protocol: EProtocol
- clickhouse_clusters: Sequence[ClickHouseCluster]
+ clickhouse_clusters: Sequence[ClickHouseCluster] = field(default_factory=list)
@dataclass
class PostgreSQLCluster:
@@ -134,6 +155,10 @@ class GenericSettings:
database: str
schema: str
- postgresql_clusters: Sequence[PostgreSQLCluster]
+ postgresql_clusters: Sequence[PostgreSQLCluster] = field(default_factory=list)
- date_time_format: EDateTimeFormat
+ @dataclass
+ class YdbCluster:
+ database: str
+
+ ydb_clusters: Sequence[YdbCluster] = field(default_factory=list)
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/types/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/types/ya.make
index abb2d281db9..a5d5840bf55 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/types/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/types/ya.make
@@ -3,6 +3,7 @@ PY3_LIBRARY()
PY_SRCS(
clickhouse.py
postgresql.py
+ ydb.py
)
END()
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/types/ydb.py b/ydb/library/yql/providers/generic/connector/tests/utils/types/ydb.py
new file mode 100644
index 00000000000..9619a224abd
--- /dev/null
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/types/ydb.py
@@ -0,0 +1,79 @@
+import abc
+
+
+class Type(abc.ABC):
+ @abc.abstractmethod
+ def to_sql(self) -> str:
+ pass
+
+
+class PrimitiveType(Type):
+ def to_sql(self):
+ return type(self).__name__
+
+ @classmethod
+ def to_nullable(cls):
+ return Nullable(cls())
+
+
+class Bool(PrimitiveType):
+ pass
+
+
+class Int8(PrimitiveType):
+ pass
+
+
+class Int16(PrimitiveType):
+ pass
+
+
+class Int32(PrimitiveType):
+ pass
+
+
+class Int64(PrimitiveType):
+ pass
+
+
+class UInt8(PrimitiveType):
+ pass
+
+
+class UInt16(PrimitiveType):
+ pass
+
+
+class UInt32(PrimitiveType):
+ pass
+
+
+class UInt64(PrimitiveType):
+ pass
+
+
+class Float(PrimitiveType):
+ pass
+
+
+class Double(PrimitiveType):
+ pass
+
+
+class String(PrimitiveType):
+ pass
+
+
+class FixedString(PrimitiveType):
+ def to_sql(self) -> str:
+ return "FixedString(5)"
+
+
+class Nullable(Type):
+ primitive: PrimitiveType
+
+ def __init__(self, primitive: PrimitiveType):
+ self.primitive = primitive
+
+ def to_sql(self) -> str:
+ return f'Nullable({self.primitive.to_sql()})'
diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/ya.make
index 03f5f71b958..4d8f719e3f1 100644
--- a/ydb/library/yql/providers/generic/connector/tests/utils/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/utils/ya.make
@@ -5,19 +5,15 @@ PY_SRCS(
comparator.py
data_source_kind.py
database.py
- dqrun.py
docker_compose.py
generate.py
- kqprun.py
log.py
- runner.py
schema.py
settings.py
sql.py
)
PEERDIR(
- contrib/python/Jinja2
contrib/python/PyYAML
ydb/library/yql/providers/generic/connector/api/common
ydb/library/yql/providers/generic/connector/tests/utils/types
@@ -28,5 +24,8 @@ PEERDIR(
END()
RECURSE_FOR_TESTS(
+ clients
+ run
+ scenario
types
)
diff --git a/ydb/library/yql/providers/generic/connector/tests/ya.make b/ydb/library/yql/providers/generic/connector/tests/ya.make
index b0d15311c50..a4d804cfdcf 100644
--- a/ydb/library/yql/providers/generic/connector/tests/ya.make
+++ b/ydb/library/yql/providers/generic/connector/tests/ya.make
@@ -1,6 +1,6 @@
RECURSE_FOR_TESTS(
+ common_test_cases
datasource
join
- test_cases
utils
)