summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ydb/tests/acceptance/cluster.yaml158
-rw-r--r--ydb/tests/acceptance/setup_slice/ya.make1
-rw-r--r--ydb/tests/acceptance/test_slice.py57
-rw-r--r--ydb/tests/acceptance/ya.make27
-rw-r--r--ydb/tests/library/harness/ydbd_slice.py120
-rw-r--r--ydb/tests/library/ya.make9
-rw-r--r--ydb/tools/ydbd_slice/handlers.py2
-rw-r--r--ydb/tools/ydbd_slice/ya.make1
8 files changed, 370 insertions, 5 deletions
diff --git a/ydb/tests/acceptance/cluster.yaml b/ydb/tests/acceptance/cluster.yaml
new file mode 100644
index 00000000000..5c3683dc5b1
--- /dev/null
+++ b/ydb/tests/acceptance/cluster.yaml
@@ -0,0 +1,158 @@
+cluster_uuid: "ydb:ydb-slice"
+static_erasure: block-4-2
+static_pdisk_type: SSD
+cms:
+ sentinel_config:
+ enable: True
+hosts:
+- name: slice-01
+ node_id: 1
+ rack: "1"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+- name: slice-02
+ node_id: 2
+ rack: "2"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+- name: slice-03
+ node_id: 3
+ rack: "3"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+- name: slice-04
+ node_id: 4
+ rack: "4"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+- name: slice-05
+ node_id: 5
+ rack: "5"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+- name: slice-06
+ node_id: 6
+ rack: "6"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+- name: slice-07
+ node_id: 7
+ rack: "7"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+- name: slice-08
+ node_id: 8
+ rack: "8"
+ drives:
+ - path: /tmp/pdisk_1.data
+ type: SSD
+ - path: /tmp/pdisk_2.data
+ type: SSD
+ - path: /tmp/pdisk_3.data
+ type: SSD
+domains:
+- domain_name: Root
+ domain_id: 1
+ plan_resolution: 10
+ dynamic_slots: 10
+ storage_pool_kinds:
+ - kind: ssd
+ geometry:
+ realm_level_begin: 10
+ realm_level_end: 20
+ domain_level_begin: 10
+ domain_level_end: 256
+ erasure: block-4-2
+ fail_domain_type: rack
+ filter_properties:
+ type: SSD
+ databases:
+ - name: db1
+ storage_units:
+ - count: 9
+ kind: ssd
+ compute_units:
+ - count: 9
+ kind: slot
+ zone: FAKE
+grpc:
+ services:
+ - yql
+ - scripting
+ - cms
+ - discovery
+ - legacy
+ - experimental
+ - pq
+ - clickhouse_internal
+ - locking
+ - pqv1
+ - pqcd
+ - topic
+ - rate_limiter
+ - monitoring
+ - datastreams
+shared_cache:
+ memory_limit: 4000000000
+pq:
+ enabled: True
+log:
+ default: 5
+ syslog: True
+ entries:
+ - level: 7
+ name: HIVE
+ - level: 7
+ name: LOCAL
+ - level: 7
+ name: BS_CONTROLLER
+ - level: 5
+ name: TX_PROXY
+ - level: 5
+ name: TX_DATASHARD
+ - level: 7
+ name: FLAT_TX_SCHEMESHARD
+ - level: 7
+ name: TENANT_SLOT_BROKER
+ - level: 6
+ name: BS_REPL
+profiles:
+- channels:
+ - storage_pool_kind: ssd
+ - storage_pool_kind: ssd
+ - storage_pool_kind: ssd
diff --git a/ydb/tests/acceptance/setup_slice/ya.make b/ydb/tests/acceptance/setup_slice/ya.make
index 0cbb8135d34..d7d2b142136 100644
--- a/ydb/tests/acceptance/setup_slice/ya.make
+++ b/ydb/tests/acceptance/setup_slice/ya.make
@@ -8,7 +8,6 @@ PY_SRCS(
PEERDIR(
ydb/tools/cfg
- ydb/tools/ydbd_slice
ydb/tests/library
)
diff --git a/ydb/tests/acceptance/test_slice.py b/ydb/tests/acceptance/test_slice.py
new file mode 100644
index 00000000000..a4fdd3da8e6
--- /dev/null
+++ b/ydb/tests/acceptance/test_slice.py
@@ -0,0 +1,57 @@
+import os
+import sys
+
+import ydb
+from ydb.tests.library.common import yatest_common
+from ydb.tests.library.harness.ydbd_slice import YdbdSlice
+
+
+class TestWithSlice(object):
+ """
+ Various tests which uses slice (e.g. host's cluster) for testing
+ """
+ @classmethod
+ def setup_class(cls):
+ cls.cluster = YdbdSlice(
+ config_path=yatest_common.source_path(os.environ["YDB_CLUSTER_YAML"]),
+ binary_path=yatest_common.binary_path(os.environ["YDB_DRIVER_BINARY"])
+ )
+ cls.cluster.start()
+
+ @classmethod
+ def teardown_class(cls):
+ cls.cluster.stop()
+
+ def test_slice_sample(self):
+ """
+ Just a sample test to ensure that slice works correctly
+ """
+ driver_config = ydb.DriverConfig(
+ database=self.cluster.db_path,
+ endpoint="%s:%s" % (
+ self.cluster.nodes[1].host, self.cluster.nodes[1].port
+ )
+ )
+ with ydb.Driver(driver_config) as driver:
+ with ydb.SessionPool(driver, size=1) as pool:
+ with pool.checkout() as session:
+ session.execute_scheme(
+ "create table `{}` (key Int32, value String, primary key(key));".format(
+ "sample_table"
+ )
+ )
+
+ def test_serializable(self):
+ yatest_common.execute(
+ [
+ yatest_common.binary_path('ydb/tests/tools/ydb_serializable/ydb_serializable'),
+ '--endpoint=%s:%d' % (self.cluster.nodes[1].host, self.cluster.nodes[1].grpc_port),
+ '--database=%s' % self.cluster.db_path,
+ '--output-path=%s' % yatest_common.output_path(),
+ '--iterations=25',
+ '--processes=1'
+ ],
+ stderr=sys.stderr,
+ wait=True,
+ stdout=sys.stdout,
+ )
diff --git a/ydb/tests/acceptance/ya.make b/ydb/tests/acceptance/ya.make
index 894766b7f62..9a7e7cd9d27 100644
--- a/ydb/tests/acceptance/ya.make
+++ b/ydb/tests/acceptance/ya.make
@@ -1,5 +1,28 @@
# Various tests which we can't run in every pull request (because of instability/specific environment/execution time/etc)
-RECURSE_ROOT_RELATIVE(
- ydb/tests/functional/large_serializable
+PY3TEST()
+ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd")
+ENV(YDB_CLUSTER_YAML="ydb/tests/acceptance/cluster.yaml")
+
+TEST_SRCS(
+ test_slice.py
)
+
+TAG(ya:fat)
+SIZE(LARGE)
+
+DEPENDS(
+ ydb/tests/tools/ydb_serializable
+ ydb/apps/ydbd
+)
+
+PEERDIR(
+ ydb/tests/library
+ ydb/tools/ydbd_slice
+)
+
+DATA(
+ arcadia/ydb/tests/acceptance/cluster.yaml
+)
+
+END()
diff --git a/ydb/tests/library/harness/ydbd_slice.py b/ydb/tests/library/harness/ydbd_slice.py
new file mode 100644
index 00000000000..f2b3f858bcb
--- /dev/null
+++ b/ydb/tests/library/harness/ydbd_slice.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import itertools
+import logging
+import time
+
+import ydb
+import ydb.tools.ydbd_slice as ydbd_slice
+from ydb.tools.cfg import walle
+
+from .kikimr_cluster import DEFAULT_INTERCONNECT_PORT, DEFAULT_MBUS_PORT, DEFAULT_MON_PORT, DEFAULT_GRPC_PORT, load_yaml
+from .kikimr_runner import KikimrExternalNode
+from .kikimr_cluster_interface import KiKiMRClusterInterface
+
+logger = logging.getLogger(__name__)
+
+
+class YdbdSlice(KiKiMRClusterInterface):
+ def __init__(self, config_path, binary_path=None):
+ kikimr_bin = binary_path
+ self.__yaml_config = load_yaml(config_path)
+ self.__hosts = [host['name'] for host in self.__yaml_config.get('hosts')]
+ ssh_user = "yc-user"
+ walle_provider = walle.NopHostsInformationProvider()
+ components = {'kikimr': ['bin', 'cfg'], 'dynamic_slots': ['all']}
+ self.cluster_details = ydbd_slice.safe_load_cluster_details(config_path, walle_provider=walle_provider)
+ self.hosts_names = self.cluster_details.hosts_names
+ nodes = ydbd_slice.nodes.Nodes(self.hosts_names, dry_run=False, ssh_user=ssh_user)
+
+ configurator = ydbd_slice.cluster_description.Configurator(
+ self.cluster_details,
+ out_dir="/tmp",
+ kikimr_bin=kikimr_bin,
+ kikimr_compressed_bin=None,
+ walle_provider=walle_provider
+ )
+
+ self.__slice = ydbd_slice.handlers.Slice(
+ components,
+ nodes,
+ self.cluster_details,
+ configurator,
+ do_clear_logs=True,
+ yav_version=None,
+ walle_provider=walle_provider,
+ )
+
+ self.db_name = list(self.cluster_details.databases.values())[0][0].name
+ self.domain = self.cluster_details.domains[0].domain_name
+ self.db_path = "/{}/{}".format(self.domain, self.db_name)
+ super(YdbdSlice, self).__init__()
+
+ @property
+ def config(self):
+ return self.__config
+
+ def add_storage_pool(self, erasure=None):
+ raise NotImplementedError()
+
+ def _wait_for_start(self):
+ driver_config = ydb.DriverConfig(
+ database=self.db_path,
+ endpoint="%s:%s" % (
+ self.nodes[1].host, self.nodes[1].port
+ )
+ )
+
+ with ydb.Driver(driver_config) as driver:
+ with ydb.SessionPool(driver, size=1) as pool:
+ with pool.checkout() as session:
+ retry_count = 60
+ # wait for readiness
+ for _ in range(retry_count):
+ try:
+ table_for_check = self.db_path + "/sample_table"
+ session.create_table(
+ table_for_check,
+ ydb.TableDescription()
+ .with_column(ydb.Column("key", ydb.PrimitiveType.Uint64))
+ .with_primary_key("key")
+ )
+ except ydb.issues.Unavailable:
+ time.sleep(1)
+ else:
+ break
+
+ session.drop_table(table_for_check)
+
+ def start(self):
+ self.__slice.slice_install()
+ self._wait_for_start()
+ return self
+
+ def stop(self):
+ self.__slice.slice_stop()
+ return self
+
+ def restart(self):
+ self.stop()
+ self.stop()
+ return self
+
+ @property
+ def nodes(self):
+ # TODO: use from config?
+ return {
+ node_id: KikimrExternalNode(
+ node_id=node_id,
+ host=host,
+ port=DEFAULT_GRPC_PORT,
+ mon_port=DEFAULT_MON_PORT,
+ ic_port=DEFAULT_INTERCONNECT_PORT,
+ mbus_port=DEFAULT_MBUS_PORT,
+ configurator=None,
+ ) for node_id, host in zip(itertools.count(start=1), self.__hosts)
+ }
+
+ @property
+ def slots(self):
+ raise NotImplementedError()
diff --git a/ydb/tests/library/ya.make b/ydb/tests/library/ya.make
index 2e043071c2b..52d44a7a563 100644
--- a/ydb/tests/library/ya.make
+++ b/ydb/tests/library/ya.make
@@ -83,6 +83,15 @@ IF (NOT PYTHON3)
)
ENDIF()
+IF (PYTHON3)
+ PEERDIR(
+ ydb/tools/ydbd_slice
+ )
+ PY_SRCS(
+ harness/ydbd_slice.py
+ )
+ENDIF()
+
PEERDIR(
contrib/python/PyHamcrest
contrib/python/PyYAML
diff --git a/ydb/tools/ydbd_slice/handlers.py b/ydb/tools/ydbd_slice/handlers.py
index d0dd4582b4a..9d14552f056 100644
--- a/ydb/tools/ydbd_slice/handlers.py
+++ b/ydb/tools/ydbd_slice/handlers.py
@@ -93,7 +93,7 @@ class Slice:
def _dynamic_configure(self):
dynamic_cfg_path = self.configurator.create_dynamic_cfg()
# wait for bs to configure
- time_remaining = 60
+ time_remaining = 120
while True:
try:
self._invoke_scripts(dynamic_cfg_path, ['init_storage.bash'])
diff --git a/ydb/tools/ydbd_slice/ya.make b/ydb/tools/ydbd_slice/ya.make
index e57056faf6d..e6d7b63cf36 100644
--- a/ydb/tools/ydbd_slice/ya.make
+++ b/ydb/tools/ydbd_slice/ya.make
@@ -23,7 +23,6 @@ PY_SRCS(
PEERDIR(
ydb/tools/cfg
ydb/public/sdk/python
- ydb/public/sdk/python/enable_v3_new_behavior
contrib/python/PyYAML
contrib/python/ruamel.yaml
contrib/python/kubernetes
generated by cgit v1.3 (git 2.53.0) at 2026-06-08 21:40:34 +0000