aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorshmel1k <shmel1k@ydb.tech>2023-11-28 12:03:45 +0300
committershmel1k <shmel1k@ydb.tech>2023-11-28 13:33:54 +0300
commit781028b5c8b17b25dae1f2bdd8f1c392e2f91e79 (patch)
treeb2a4f59a7168b814ac6bbe70646fc83882f4514a
parent31cf81679842e3335cbcfeae3a52bd4cbb829c9a (diff)
downloadydb-781028b5c8b17b25dae1f2bdd8f1c392e2f91e79.tar.gz
improve styling for ydb_configure
-rw-r--r--ydb/tools/cfg/base.py238
-rw-r--r--ydb/tools/cfg/bin/__main__.py53
-rw-r--r--ydb/tools/cfg/configurator_setup.py87
-rw-r--r--ydb/tools/cfg/dynamic.py102
-rw-r--r--ydb/tools/cfg/static.py267
-rw-r--r--ydb/tools/cfg/templates.py261
-rw-r--r--ydb/tools/cfg/types.py20
-rw-r--r--ydb/tools/cfg/utils.py31
-rw-r--r--ydb/tools/cfg/validation.py486
-rw-r--r--ydb/tools/cfg/walle/__init__.py4
-rw-r--r--ydb/tools/cfg/walle/walle.py2
11 files changed, 665 insertions, 886 deletions
diff --git a/ydb/tools/cfg/base.py b/ydb/tools/cfg/base.py
index ae96212e59..f139cfba32 100644
--- a/ydb/tools/cfg/base.py
+++ b/ydb/tools/cfg/base.py
@@ -26,7 +26,11 @@ DEFAULT_BOX_ID = 1
VDISKS_DEFAULT_CONFIG = {
'vdisk_kinds': [
{'kind': 'Log', 'base_kind': 'Default', 'config': {'fresh_use_dreg': True, 'allow_keep_flags': False}},
- {'kind': 'LocalMode', 'base_kind': 'Default', 'config': {'hull_comp_level0_max_ssts_at_once': 2, 'hull_comp_sorted_parts_num': 1}},
+ {
+ 'kind': 'LocalMode',
+ 'base_kind': 'Default',
+ 'config': {'hull_comp_level0_max_ssts_at_once': 2, 'hull_comp_sorted_parts_num': 1},
+ },
]
}
@@ -42,9 +46,7 @@ GRPC_DEFAULT_CONFIG = {
'keep_alive_probe_interval_sec': 10,
}
-DYNAMIC_NAME_SERVICE = {
- 'max_static_node_id': 50000
-}
+DYNAMIC_NAME_SERVICE = {'max_static_node_id': 50000}
IC_DEFAULT_CONFIG = {
'start_tcp': True,
@@ -67,19 +69,22 @@ class KiKiMRDrive(object):
def __eq__(self, other):
return (
- self.type == other.type and
- self.path == other.path and
- self.shared_with_os == other.shared_with_os and
- self.expected_slot_count == other.expected_slot_count and
- self.kind == other.kind
+ self.type == other.type
+ and self.path == other.path
+ and self.shared_with_os == other.shared_with_os
+ and self.expected_slot_count == other.expected_slot_count
+ and self.kind == other.kind
)
def __hash__(self):
- return hash('\0'.join(map(str, (self.type, self.path, self.shared_with_os, self.expected_slot_count, self.kind))))
+ return hash(
+ '\0'.join(map(str, (self.type, self.path, self.shared_with_os, self.expected_slot_count, self.kind)))
+ )
Domain = collections.namedtuple(
- '_Domain', [
+ '_Domain',
+ [
'domain_name',
'domain_id',
'plan_resolution',
@@ -92,38 +97,34 @@ Domain = collections.namedtuple(
'tenants',
'console_initializers',
'bind_slots_to_numa_nodes',
- 'config_cookie'
- ]
+ 'config_cookie',
+ ],
)
KiKiMRHost = collections.namedtuple(
- '_KiKiMRHost', [
- 'hostname',
- 'node_id',
- 'drives',
- 'ic_port',
- 'body',
- 'datacenter',
- 'rack',
- 'host_config_id'
- ]
+ '_KiKiMRHost', ['hostname', 'node_id', 'drives', 'ic_port', 'body', 'datacenter', 'rack', 'host_config_id']
)
DEFAULT_PLAN_RESOLUTION = 10
-DEFAULT_DOMAIN = {
- 'domain_name': 'Root',
- 'domain_id': 1,
- 'plan_resolution': DEFAULT_PLAN_RESOLUTION
-}
+DEFAULT_DOMAIN = {'domain_name': 'Root', 'domain_id': 1, 'plan_resolution': DEFAULT_PLAN_RESOLUTION}
DEFAULT_RESOURCE_BROKER_CPU_LIMIT = 20
-DEFAULT_RESOURCE_BROKER_MEMORY_LIMIT = 16 * (1024 ** 3)
+DEFAULT_RESOURCE_BROKER_MEMORY_LIMIT = 16 * (1024**3)
class StoragePool(object):
def __init__(
- self, box_id, erasure, num_groups, filter_properties,
- fail_domain_type, kind, name, vdisk_kind, encryption_mode, generation=0
+ self,
+ box_id,
+ erasure,
+ num_groups,
+ filter_properties,
+ fail_domain_type,
+ kind,
+ name,
+ vdisk_kind,
+ encryption_mode,
+ generation=0,
):
self._dict_items = {}
self.box_id = box_id
@@ -151,10 +152,7 @@ class StoragePool(object):
class StoragePoolKind(object):
- def __init__(
- self, box_id, erasure, filter_properties,
- fail_domain_type, kind, vdisk_kind, encryption_mode
- ):
+ def __init__(self, box_id, erasure, filter_properties, fail_domain_type, kind, vdisk_kind, encryption_mode):
self._dict_items = {}
self.box_id = box_id
self.erasure = erasure
@@ -178,36 +176,32 @@ class StoragePoolKind(object):
Subdomain = collections.namedtuple(
- '_Subdomain', [
+ '_Subdomain',
+ [
'domain',
'name',
'plan_resolution',
'mediators',
'coordinators',
'storage_pools',
- ]
+ ],
)
DynamicSlot = collections.namedtuple(
- '_DynamicSlot', [
+ '_DynamicSlot',
+ [
'domain',
'id',
- ]
+ ],
)
Profile = collections.namedtuple(
- '_Profile', [
+ '_Profile',
+ [
'channels',
- ]
-)
-Channel = collections.namedtuple(
- '_Channel', [
- 'erasure',
- 'pdisk_type',
- 'vdisk_kind',
- 'storage_pool_kind'
- ]
+ ],
)
+Channel = collections.namedtuple('_Channel', ['erasure', 'pdisk_type', 'vdisk_kind', 'storage_pool_kind'])
class StorageUnit(object):
@@ -224,19 +218,15 @@ class ComputeUnit(object):
class Tenant(object):
- def __init__(self, name, storage_units, compute_units=None, overridden_configs=None, shared=False, plan_resolution=None):
+ def __init__(
+ self, name, storage_units, compute_units=None, overridden_configs=None, shared=False, plan_resolution=None
+ ):
self.name = name
self.overridden_configs = overridden_configs
- self.storage_units = tuple(
- StorageUnit(**storage_unit_template)
- for storage_unit_template in storage_units
- )
+ self.storage_units = tuple(StorageUnit(**storage_unit_template) for storage_unit_template in storage_units)
self.compute_units = tuple()
if compute_units:
- self.compute_units = tuple(
- ComputeUnit(**compute_unit_template)
- for compute_unit_template in compute_units
- )
+ self.compute_units = tuple(ComputeUnit(**compute_unit_template) for compute_unit_template in compute_units)
self.shared = shared
self.plan_resolution = plan_resolution
@@ -274,14 +264,11 @@ def normalize_domain(domain_name):
class ClusterDetailsProvider(object):
def __init__(self, template, walle_provider, database=None):
- self.__cluster_description = _ClusterDescription.from_template(
- template)
+ self.__cluster_description = _ClusterDescription.from_template(template)
self.__cluster_description.validate()
if database is not None:
- self.__cluster_description = self.get_subjective_description(
- self.__cluster_description,
- database)
+ self.__cluster_description = self.get_subjective_description(self.__cluster_description, database)
self._use_walle = self.__cluster_description.get('use_walle', True)
if not walle_provider:
@@ -318,17 +305,18 @@ class ClusterDetailsProvider(object):
if get_full_tenant_name(domain_of_tenant, current_tenant.get('name')) != tenant:
continue
- overridden_configurations = current_tenant.get(
- 'overridden_configs', {})
- for service in ('nfs', 'nfs_control', 'nbs', 'nbs_control', ):
+ overridden_configurations = current_tenant.get('overridden_configs', {})
+ for service in (
+ 'nfs',
+ 'nfs_control',
+ 'nbs',
+ 'nbs_control',
+ ):
if service in overridden_configurations:
- overridden_configurations[service]['domain'] = domain_template.get(
- 'domain_name')
- overridden_configurations[service]['subdomain'] = current_tenant.get(
- 'name')
+ overridden_configurations[service]['domain'] = domain_template.get('domain_name')
+ overridden_configurations[service]['subdomain'] = current_tenant.get('name')
- subjective_description = _ClusterDescription(
- dict(objective_description))
+ subjective_description = _ClusterDescription(dict(objective_description))
subjective_description.update(overridden_configurations)
subjective_description.validate()
return subjective_description
@@ -400,19 +388,11 @@ class ClusterDetailsProvider(object):
return self._hosts
futures = []
for node_id, host_description in enumerate(self.__cluster_description.get('hosts'), 1):
- futures.append(
- self._thread_pool.submit(
- self.__collect_host_info,
- node_id,
- host_description
- )
- )
+ futures.append(self._thread_pool.submit(self.__collect_host_info, node_id, host_description))
r = []
for f in futures:
- r.append(
- f.result()
- )
+ r.append(f.result())
if self._hosts is None:
self._hosts = r
@@ -420,9 +400,7 @@ class ClusterDetailsProvider(object):
@property
def system_tablets_node_ids(self):
- return [
- host.node_id for host in self.static_bs_group_hosts
- ]
+ return [host.node_id for host in self.static_bs_group_hosts]
@property
def state_storages(self):
@@ -434,8 +412,7 @@ class ClusterDetailsProvider(object):
if node_ids:
return node_ids
- state_storage_node_set = set(self.__cluster_description.get(
- 'state_storage', {}).get('node_set', []))
+ state_storage_node_set = set(self.__cluster_description.get('state_storage', {}).get('node_set', []))
if len(state_storage_node_set) < 1:
return [host.node_id for host in self.static_bs_group_hosts]
node_ids = []
@@ -450,8 +427,7 @@ class ClusterDetailsProvider(object):
@property
def static_bs_group_hosts(self):
- static_bs_group_filter = self.__cluster_description.get(
- 'static_bs_group_hosts', [])
+ static_bs_group_filter = self.__cluster_description.get('static_bs_group_hosts', [])
matched = []
for host in self.hosts:
if host.hostname in static_bs_group_filter or len(static_bs_group_filter) == 0:
@@ -491,9 +467,7 @@ class ClusterDetailsProvider(object):
HostConfig(
host_config_id=host_config['host_config_id'],
generation=host_config.get('generation', 0),
- drives=tuple(
- KiKiMRDrive(**drive) for drive in host_config_drives
- )
+ drives=tuple(KiKiMRDrive(**drive) for drive in host_config_drives),
)
)
return converted_host_configs
@@ -512,17 +486,14 @@ class ClusterDetailsProvider(object):
for channel in profile.get('channels', []):
erasure = channel.get('erasure', str(self.static_erasure))
pdisk_type = channel.get('pdisk_type')
- pdisk_type = None if pdisk_type is None else types.PDiskCategory.from_string(
- pdisk_type)
- channel_storage_pool_kind = channel.get(
- 'storage_pool_kind', None)
+ pdisk_type = None if pdisk_type is None else types.PDiskCategory.from_string(pdisk_type)
+ channel_storage_pool_kind = channel.get('storage_pool_kind', None)
if channel_storage_pool_kind is not None and pdisk_type is None:
pdisk_type = None
for storage_pool_kind in storage_pool_kinds:
if channel_storage_pool_kind == storage_pool_kind.kind:
f_properties = storage_pool_kind.filter_properties
- pdisk_type = types.PDiskCategory.from_string(
- f_properties['type'])
+ pdisk_type = types.PDiskCategory.from_string(f_properties['type'])
assert pdisk_type is not None
@@ -534,13 +505,7 @@ class ClusterDetailsProvider(object):
storage_pool_kind=channel_storage_pool_kind,
)
)
- all_profiles.append(
- Profile(
- channels=tuple(
- profile_channels
- )
- )
- )
+ all_profiles.append(Profile(channels=tuple(profile_channels)))
return tuple(all_profiles)
# Domains Stuff
@@ -552,8 +517,7 @@ class ClusterDetailsProvider(object):
box_id=storage_description.get("box_id", DEFAULT_BOX_ID),
erasure=storage_description.get('erasure'),
filter_properties=storage_description.get('filter_properties', {}),
- fail_domain_type=storage_description.get(
- 'fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE),
+ fail_domain_type=storage_description.get('fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE),
vdisk_kind=storage_description.get('vdisk_kind', 'Default'),
encryption_mode=storage_description.get('encryption_mode', 0),
)
@@ -572,8 +536,7 @@ class ClusterDetailsProvider(object):
@property
def domains(self):
- cl_desc_domains = self.__cluster_description.get(
- 'domains', [DEFAULT_DOMAIN])
+ cl_desc_domains = self.__cluster_description.get('domains', [DEFAULT_DOMAIN])
domains = []
for domain_id, domain in enumerate(cl_desc_domains, 1):
domain_name = domain.get('domain_name')
@@ -582,12 +545,12 @@ class ClusterDetailsProvider(object):
pool_kind.get('kind'): self.__storage_pool_kind(pool_kind)
for pool_kind in domain.get('storage_pool_kinds', [])
}
- assert len(set(storage_pool_kinds.keys())) == len(storage_pool_kinds.keys()), \
- "required unique kind value in storage_pool_kinds items"
+ assert len(set(storage_pool_kinds.keys())) == len(
+ storage_pool_kinds.keys()
+ ), "required unique kind value in storage_pool_kinds items"
storage_pools = [
- self.__storage_pool(storage_pool_kinds,
- pool_instance, domain_name)
+ self.__storage_pool(storage_pool_kinds, pool_instance, domain_name)
for pool_instance in domain.get('storage_pools', [])
]
@@ -595,14 +558,10 @@ class ClusterDetailsProvider(object):
Domain(
domain_name=domain_name,
domain_id=domain.get('domain_id', domain_id),
- mediators=domain.get(
- 'mediators', self.__coordinators_count_optimal),
- coordinators=domain.get(
- 'coordinators', self.__mediators_count_optimal),
- allocators=domain.get(
- 'allocators', self.__allocators_count_optimal),
- plan_resolution=domain.get(
- 'plan_resolution', DEFAULT_PLAN_RESOLUTION),
+ mediators=domain.get('mediators', self.__coordinators_count_optimal),
+ coordinators=domain.get('coordinators', self.__mediators_count_optimal),
+ allocators=domain.get('allocators', self.__allocators_count_optimal),
+ plan_resolution=domain.get('plan_resolution', DEFAULT_PLAN_RESOLUTION),
dynamic_slots=domain.get('dynamic_slots', 0),
storage_pool_kinds=storage_pool_kinds,
storage_pools=storage_pools,
@@ -619,8 +578,7 @@ class ClusterDetailsProvider(object):
tenants = domain_description.get('databases', [])
prepared_tenants = []
for tenant in tenants:
- prepared_tenants.append(
- Tenant(**tenant))
+ prepared_tenants.append(Tenant(**tenant))
return prepared_tenants
# Static BlobStorage Group Stuff
@@ -636,10 +594,7 @@ class ClusterDetailsProvider(object):
@property
def fail_domain_type(self):
return types.FailDomainType.from_string(
- str(
- self.__cluster_description.get(
- 'fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE)
- )
+ str(self.__cluster_description.get('fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE))
)
@property
@@ -648,8 +603,7 @@ class ClusterDetailsProvider(object):
@property
def static_erasure(self):
- return types.Erasure.from_string(
- self.__cluster_description['static_erasure'])
+ return types.Erasure.from_string(self.__cluster_description['static_erasure'])
# Auth txt
@property
@@ -780,14 +734,12 @@ class ClusterDetailsProvider(object):
box_id=DEFAULT_BOX_ID,
erasure=storage_pool.get('erasure'),
num_groups=storage_pool.get('num_groups'),
- filter_properties=storage_pool.get(
- 'filter_properties', {}),
- fail_domain_type=storage_pool.get(
- 'fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE),
+ filter_properties=storage_pool.get('filter_properties', {}),
+ fail_domain_type=storage_pool.get('fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE),
kind=storage_pool.get('kind', None),
vdisk_kind=storage_pool.get('vdisk_kind', 'Default'),
encryption_mode=storage_pool.get('encryption_mode', 0),
- generation=storage_pool.get('generation', 0)
+ generation=storage_pool.get('generation', 0),
)
)
@@ -802,14 +754,11 @@ class ClusterDetailsProvider(object):
def __storage_pool(self, storage_pool_kinds, storage_pool_instance, domain_name):
kind_name = storage_pool_instance.get('kind')
- assert kind_name in storage_pool_kinds, \
- "databse storage pools should use pool kinds from storage_pool_kinds section"
+ assert (
+ kind_name in storage_pool_kinds
+ ), "databse storage pools should use pool kinds from storage_pool_kinds section"
- storage_pool_name = "/{domain}:{kind}".format(
- domain=domain_name,
- kind=kind_name
-
- )
+ storage_pool_name = "/{domain}:{kind}".format(domain=domain_name, kind=kind_name)
return StoragePool(
name=storage_pool_name,
@@ -823,12 +772,7 @@ class ClusterDetailsProvider(object):
slots = []
for domain_description in self.__cluster_description.get('domains', []):
for slot_id in range(1, int(domain_description.get('dynamic_slots', 0) + 1)):
- slots.append(
- DynamicSlot(
- domain=domain_description.get('domain_name'),
- id=slot_id
- )
- )
+ slots.append(DynamicSlot(domain=domain_description.get('domain_name'), id=slot_id))
return slots
@property
diff --git a/ydb/tools/cfg/bin/__main__.py b/ydb/tools/cfg/bin/__main__.py
index 7d50e8ffdc..a4c6156311 100644
--- a/ydb/tools/cfg/bin/__main__.py
+++ b/ydb/tools/cfg/bin/__main__.py
@@ -7,35 +7,30 @@ from logging import config as logging_config
import yaml
-from ydb.tools.cfg.configurator_setup import (get_parser,
- parse_optional_arguments)
+from ydb.tools.cfg.configurator_setup import get_parser, parse_optional_arguments
from ydb.tools.cfg.dynamic import DynamicConfigGenerator
from ydb.tools.cfg.static import StaticConfigGenerator
from ydb.tools.cfg.utils import write_to_file
-logging_config.dictConfig({
- 'version': 1,
- 'formatters': {
- 'base': {
- 'format': '%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s',
+logging_config.dictConfig(
+ {
+ 'version': 1,
+ 'formatters': {
+ 'base': {
+ 'format': '%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s',
+ },
},
- },
- 'handlers': {
- 'console': {
- 'class': 'logging.StreamHandler',
- 'level': 'INFO',
- 'formatter': 'base',
- 'stream': sys.stdout,
+ 'handlers': {
+ 'console': {
+ 'class': 'logging.StreamHandler',
+ 'level': 'INFO',
+ 'formatter': 'base',
+ 'stream': sys.stdout,
+ },
},
- },
- 'root': {
- 'formatter': 'base',
- 'level': 'INFO',
- 'handlers': (
- 'console',
- )
- },
-})
+ 'root': {'formatter': 'base', 'level': 'INFO', 'handlers': ('console',)},
+ }
+)
logger = logging.getLogger()
@@ -52,19 +47,11 @@ def cfg_generate(args):
with open(args.cluster_description, 'r') as yaml_template:
cluster_template = yaml.safe_load(yaml_template)
- generator = cfg_cls(
- cluster_template,
- args.binary_path,
- args.output_dir,
- **kwargs
- )
+ generator = cfg_cls(cluster_template, args.binary_path, args.output_dir, **kwargs)
all_configs = generator.get_all_configs()
for cfg_name, cfg_value in all_configs.items():
- write_to_file(
- os.path.join(args.output_dir, cfg_name),
- cfg_value
- )
+ write_to_file(os.path.join(args.output_dir, cfg_name), cfg_value)
def main():
diff --git a/ydb/tools/cfg/configurator_setup.py b/ydb/tools/cfg/configurator_setup.py
index 18041efcc7..981bdd7b5d 100644
--- a/ydb/tools/cfg/configurator_setup.py
+++ b/ydb/tools/cfg/configurator_setup.py
@@ -39,66 +39,33 @@ def get_parser(generate_func, extra_cfg_arguments=[]):
)
subparsers = parser.add_subparsers(help='sub-command help')
- parser_cfg = subparsers.add_parser(
- 'cfg', help='run static config generation')
+ parser_cfg = subparsers.add_parser('cfg', help='run static config generation')
parser_cfg.add_argument(
- "cluster_description", type=str,
+ "cluster_description",
+ type=str,
help="Path to the cluster description",
)
+ parser_cfg.add_argument("binary_path", type=str, help="Path to the KiKiMR's binary")
+ parser_cfg.add_argument("output_dir", type=str, help="Path to store configuration files")
+ parser_cfg.add_argument("--seed", type=int, help='Force cfg command to use special seed in instantiate random')
+ parser_cfg.add_argument("--grpc-endpoint", type=str, help='gRPC endpoint to use in scripts')
parser_cfg.add_argument(
- "binary_path", type=str,
- help="Path to the KiKiMR's binary"
+ '--database', type=str, help='Database to serve (possible options: name of domain or no for pure storage nodes)'
)
parser_cfg.add_argument(
- "output_dir", type=str,
- help="Path to store configuration files"
+ '--tenant', type=str, help='Tenant to serve (possible options: name of domain or no for pure storage nodes)'
)
+ parser_cfg.add_argument('--enable-cores', action='store_true', help='Enables coredumps')
parser_cfg.add_argument(
- "--seed", type=int,
- help='Force cfg command to use special seed in instantiate random'
- )
- parser_cfg.add_argument(
- "--grpc-endpoint", type=str,
- help='gRPC endpoint to use in scripts'
- )
- parser_cfg.add_argument(
- '--database', type=str,
- help='Database to serve (possible options: name of domain or no for pure storage nodes)'
- )
- parser_cfg.add_argument(
- '--tenant', type=str,
- help='Tenant to serve (possible options: name of domain or no for pure storage nodes)'
- )
- parser_cfg.add_argument(
- '--enable-cores', action='store_true',
- help='Enables coredumps'
- )
- parser_cfg.add_argument(
- '--dynamic-node', action='store_true',
- help='Indicates that configuration should be generated for dynamic node'
- )
- parser_cfg.add_argument(
- '--node-broker-port', type=str,
- help='Node Broker Port to use'
- )
- parser_cfg.add_argument(
- '--cfg-home', type=str,
- help='Configuration home directory'
- )
- parser_cfg.add_argument(
- '--binaries-home', type=str,
- help='Binaries home directory'
- )
- parser_cfg.add_argument(
- '--local-binary-path', type=str,
- help='Path to kikimr binary current host'
+ '--dynamic-node', action='store_true', help='Indicates that configuration should be generated for dynamic node'
)
+ parser_cfg.add_argument('--node-broker-port', type=str, help='Node Broker Port to use')
+ parser_cfg.add_argument('--cfg-home', type=str, help='Configuration home directory')
+ parser_cfg.add_argument('--binaries-home', type=str, help='Binaries home directory')
+ parser_cfg.add_argument('--local-binary-path', type=str, help='Path to kikimr binary current host')
for port in ('grpc-port', 'ic-port', 'mon-port', 'mbus-port', 'sqs-port'):
- parser_cfg.add_argument(
- '--%s' % port, type=str,
- help='Port to be used during KiKiMR server start'
- )
+ parser_cfg.add_argument('--%s' % port, type=str, help='Port to be used during KiKiMR server start')
for v in extra_cfg_arguments:
parser_cfg.add_argument(
@@ -109,28 +76,18 @@ def get_parser(generate_func, extra_cfg_arguments=[]):
argument_group = parser_cfg.add_mutually_exclusive_group()
argument_group.add_argument(
- '--static', action='store_true',
- help='Forces cfg command to generate static configuration'
- )
- argument_group.add_argument(
- '--dynamic', action='store_true',
- help='Forces cfg command to generate dynamic configuration'
- )
- argument_group.add_argument(
- '--nbs', action='store_true',
- help='Forces cfg command to generate NBS configuration'
+ '--static', action='store_true', help='Forces cfg command to generate static configuration'
)
argument_group.add_argument(
- '--nbs-control', action='store_true',
- help='Forces cfg command to generate NBS Control configuration'
+ '--dynamic', action='store_true', help='Forces cfg command to generate dynamic configuration'
)
+ argument_group.add_argument('--nbs', action='store_true', help='Forces cfg command to generate NBS configuration')
argument_group.add_argument(
- '--nfs', action='store_true',
- help='Forces cfg command to generate NFS configuration'
+ '--nbs-control', action='store_true', help='Forces cfg command to generate NBS Control configuration'
)
+ argument_group.add_argument('--nfs', action='store_true', help='Forces cfg command to generate NFS configuration')
argument_group.add_argument(
- '--nfs-control', action='store_true',
- help='Forces cfg command to generate NFS Control configuration'
+ '--nfs-control', action='store_true', help='Forces cfg command to generate NFS Control configuration'
)
parser_cfg.set_defaults(func=generate_func)
diff --git a/ydb/tools/cfg/dynamic.py b/ydb/tools/cfg/dynamic.py
index 8bd8aebc85..f786b654a0 100644
--- a/ydb/tools/cfg/dynamic.py
+++ b/ydb/tools/cfg/dynamic.py
@@ -4,21 +4,24 @@ import itertools
import os
from contrib.ydb.core.protos import blobstorage_config_pb2 as bs_config
-from contrib.ydb.core.protos import \
- blobstorage_pdisk_config_pb2 as pdisk_config
+from contrib.ydb.core.protos import blobstorage_pdisk_config_pb2 as pdisk_config
from contrib.ydb.core.protos import flat_scheme_op_pb2 as flat_scheme_op
from contrib.ydb.core.protos import msgbus_pb2 as msgbus
from contrib.ydb.core.protos import tx_proxy_pb2 as tx_proxy
from ydb.tools.cfg import base, static, utils
-from ydb.tools.cfg.types import (DistinctionLevels, Erasure, FailDomainType,
- PDiskCategory)
+from ydb.tools.cfg.types import DistinctionLevels, Erasure, FailDomainType, PDiskCategory
class DynamicConfigGenerator(object):
def __init__(
- self, template, binary_path, output_dir, grpc_endpoint=None,
- local_binary_path=None, walle_provider=None,
- **kwargs
+ self,
+ template,
+ binary_path,
+ output_dir,
+ grpc_endpoint=None,
+ local_binary_path=None,
+ walle_provider=None,
+ **kwargs
):
self._template = template
self._binary_path = binary_path
@@ -29,7 +32,9 @@ class DynamicConfigGenerator(object):
self._grpc_endpoint = grpc_endpoint
self.__define_storage_pools_request = None
self.__configure_request = None
- self.__static_config = static.StaticConfigGenerator(template, binary_path, output_dir, walle_provider=walle_provider, local_binary_path=local_binary_path)
+ self.__static_config = static.StaticConfigGenerator(
+ template, binary_path, output_dir, walle_provider=walle_provider, local_binary_path=local_binary_path
+ )
@property
def grpc_endpoint(self):
@@ -49,16 +54,23 @@ class DynamicConfigGenerator(object):
"config",
"invoke",
"--proto-file",
- os.path.join(
- self._output_dir,
- proto_file
- )
+ os.path.join(self._output_dir, proto_file),
]
)
def __add_storage_pool(
- self, box_id, storage_pool_id, erasure, filter_properties, num_groups, fail_domain_type, kind=None,
- name=None, vdisk_kind='Default', encryption_mode=0, generation=0,
+ self,
+ box_id,
+ storage_pool_id,
+ erasure,
+ filter_properties,
+ num_groups,
+ fail_domain_type,
+ kind=None,
+ name=None,
+ vdisk_kind='Default',
+ encryption_mode=0,
+ generation=0,
):
cmd = self.__define_storage_pools_request.Command.add()
cmd.DefineStoragePool.BoxId = box_id
@@ -104,10 +116,7 @@ class DynamicConfigGenerator(object):
"db",
"schema",
"execute",
- os.path.join(
- self._output_dir,
- proto_file
- ),
+ os.path.join(self._output_dir, proto_file),
]
)
@@ -122,10 +131,7 @@ class DynamicConfigGenerator(object):
"execute",
"--domain=%s" % domain_name,
"--retry=10",
- os.path.join(
- self._output_dir,
- proto_file
- ),
+ os.path.join(self._output_dir, proto_file),
]
)
@@ -140,34 +146,29 @@ class DynamicConfigGenerator(object):
for init_id, filename in enumerate(domain.console_initializers, 1):
commands.append(self.__cms_init_cmd('Configure-%s-init-%d.txt' % (name, init_id), name))
- return '\n'.join(
- commands
- )
+ return '\n'.join(commands)
def init_storage_commands(self):
return '\n'.join(
[
"set -eu",
self.__init_storage_command("DefineBox.txt"),
- self.__init_storage_command("DefineStoragePools.txt")
+ self.__init_storage_command("DefineStoragePools.txt"),
]
)
def init_compute_commands(self):
commands = ["set -eu"]
- return '\n'.join(
- commands
- )
+ return '\n'.join(commands)
def init_root_storage(self):
commands = ["set -eu"]
commands += [
self.__bind_storage_with_root('BindRootStorageRequest-%s.txt' % domain.domain_name)
- for domain in self._cluster_details.domains if len(domain.storage_pools)
+ for domain in self._cluster_details.domains
+ if len(domain.storage_pools)
]
- return '\n'.join(
- commands
- )
+ return '\n'.join(commands)
@property
def cms_init_cmd(self):
@@ -207,7 +208,8 @@ class DynamicConfigGenerator(object):
if at_least_one_host_config_defined:
raise RuntimeError(
- "At least one host config defined manually, but you still use drives directly attached to hosts")
+ "At least one host config defined manually, but you still use drives directly attached to hosts"
+ )
host_config_id = next(host_config_id_iter)
cmd = define_box_request.Command.add()
@@ -236,19 +238,13 @@ class DynamicConfigGenerator(object):
if self._cluster_details.storage_pools_deprecated:
for storage_pool in self._cluster_details.storage_pools_deprecated:
- self.__add_storage_pool(
- storage_pool_id=next(storage_pool_id),
- **storage_pool.to_dict()
- )
+ self.__add_storage_pool(storage_pool_id=next(storage_pool_id), **storage_pool.to_dict())
# for tablets in domain lets make pools
# but it is not supposed to make tablets in domain directly
for domain in self._cluster_details.domains:
for storage_pool in domain.storage_pools:
- self.__add_storage_pool(
- storage_pool_id=next(storage_pool_id),
- **storage_pool.to_dict()
- )
+ self.__add_storage_pool(storage_pool_id=next(storage_pool_id), **storage_pool.to_dict())
return self.__define_storage_pools_request
@@ -283,7 +279,9 @@ class DynamicConfigGenerator(object):
if domain.config_cookie:
action.AddConfigItem.ConfigItem.Cookie = domain.config_cookie
- configure_request.ConfigureRequest.Actions.add().RemoveConfigItems.CookieFilter.Cookies.append(domain.config_cookie)
+ configure_request.ConfigureRequest.Actions.add().RemoveConfigItems.CookieFilter.Cookies.append(
+ domain.config_cookie
+ )
action = configure_request.ConfigureRequest.Actions.add()
action.AddConfigItem.ConfigItem.UsageScope.TenantAndNodeTypeFilter.Tenant = "dynamic"
@@ -306,8 +304,7 @@ class DynamicConfigGenerator(object):
@staticmethod
def _construct_create_tenant_request(domain, tenant):
console_request = msgbus.TConsoleRequest()
- console_request.CreateTenantRequest.Request.path = os.path.join(
- '/', domain.domain_name, tenant.name)
+ console_request.CreateTenantRequest.Request.path = os.path.join('/', domain.domain_name, tenant.name)
if tenant.shared:
resources = console_request.CreateTenantRequest.Request.shared_resources
@@ -337,8 +334,7 @@ class DynamicConfigGenerator(object):
for domain in self._cluster_details.domains:
for tenant in domain.tenants:
tenants_count += 1
- files['CreateTenant-%d.txt' % next(tn_id)] = self._construct_create_tenant_request(
- domain, tenant)
+ files['CreateTenant-%d.txt' % next(tn_id)] = self._construct_create_tenant_request(domain, tenant)
return tenants_count, files
def get_create_tenant_commands(self):
@@ -346,10 +342,7 @@ class DynamicConfigGenerator(object):
commands = []
for domain in self._cluster_details.domains:
for _ in domain.tenants:
- commands.append(
- self.__cms_init_cmd(
- 'CreateTenant-%d.txt' % next(tn_id),
- domain.domain_name))
+ commands.append(self.__cms_init_cmd('CreateTenant-%d.txt' % next(tn_id), domain.domain_name))
commands.append('exit 0')
return "\n".join(commands)
@@ -358,9 +351,10 @@ class DynamicConfigGenerator(object):
all_configs = self.get_storage_requests()
all_configs.update(
{
- 'BindRootStorageRequest-%s.txt' % domain.domain_name: utils.message_to_string(
- self.make_bind_root_storage_request(domain))
- for domain in self._cluster_details.domains if len(domain.storage_pools)
+ 'BindRootStorageRequest-%s.txt'
+ % domain.domain_name: utils.message_to_string(self.make_bind_root_storage_request(domain))
+ for domain in self._cluster_details.domains
+ if len(domain.storage_pools)
}
)
all_configs.update(
diff --git a/ydb/tools/cfg/static.py b/ydb/tools/cfg/static.py
index b07f3c9bff..f9da81776a 100644
--- a/ydb/tools/cfg/static.py
+++ b/ydb/tools/cfg/static.py
@@ -8,32 +8,50 @@ import subprocess
import tempfile
import yaml
-from contrib.ydb.core.fq.libs.config.protos.fq_config_pb2 import \
- TConfig as TFederatedQueryConfig
+from contrib.ydb.core.fq.libs.config.protos.fq_config_pb2 import TConfig as TFederatedQueryConfig
from google.protobuf import json_format
-from contrib.ydb.core.protos import (auth_pb2, blobstorage_vdisk_config_pb2,
- cms_pb2, config_pb2, feature_flags_pb2,
- netclassifier_pb2, pqconfig_pb2,
- resource_broker_pb2)
+from contrib.ydb.core.protos import (
+ auth_pb2,
+ blobstorage_vdisk_config_pb2,
+ cms_pb2,
+ config_pb2,
+ feature_flags_pb2,
+ netclassifier_pb2,
+ pqconfig_pb2,
+ resource_broker_pb2,
+)
from ydb.tools.cfg import base, types, utils
-from ydb.tools.cfg.templates import (dynamic_cfg_new_style,
- kikimr_cfg_for_dynamic_node,
- kikimr_cfg_for_dynamic_slot,
- kikimr_cfg_for_static_node,
- kikimr_cfg_for_static_node_new_style)
+from ydb.tools.cfg.templates import (
+ dynamic_cfg_new_style,
+ kikimr_cfg_for_dynamic_node,
+ kikimr_cfg_for_dynamic_slot,
+ kikimr_cfg_for_static_node,
+ kikimr_cfg_for_static_node_new_style,
+)
logger = logging.getLogger(__name__)
class StaticConfigGenerator(object):
def __init__(
- self, template, binary_path, output_dir, database=None,
- node_broker_port=2135, ic_port=19001, walle_provider=None,
- grpc_port=2135, mon_port=8765, cfg_home='/Berkanavt/kikimr',
- sqs_port=8771, enable_cores=False, enable_cms_config_cache=False,
- local_binary_path=None, skip_location=False,
- **kwargs
+ self,
+ template,
+ binary_path,
+ output_dir,
+ database=None,
+ node_broker_port=2135,
+ ic_port=19001,
+ walle_provider=None,
+ grpc_port=2135,
+ mon_port=8765,
+ cfg_home='/Berkanavt/kikimr',
+ sqs_port=8771,
+ enable_cores=False,
+ enable_cms_config_cache=False,
+ local_binary_path=None,
+ skip_location=False,
+ **kwargs
):
self.__proto_configs = {}
self.__binary_path = binary_path
@@ -81,16 +99,23 @@ class StaticConfigGenerator(object):
'pqcd.txt': None,
'failure_injection.txt': None,
}
- self.__optional_config_files = set((
- 'rb.txt',
- 'metering.txt',
- 'audit.txt',
- 'fq.txt',
- 'failure_injection.txt',
- ))
+ self.__optional_config_files = set(
+ (
+ 'rb.txt',
+ 'metering.txt',
+ 'audit.txt',
+ 'fq.txt',
+ 'failure_injection.txt',
+ )
+ )
self._enable_cms_config_cache = template.get('enable_cms_config_cache', enable_cms_config_cache)
if 'tracing' in template:
- self.__tracing = (template['tracing']['host'], template['tracing']['port'], template['tracing']['root_ca'], template['tracing']['service_name'])
+ self.__tracing = (
+ template['tracing']['host'],
+ template['tracing']['port'],
+ template['tracing']['root_ca'],
+ template['tracing']['service_name'],
+ )
else:
self.__tracing = None
self.__write_mbus_settings_to_kikimr_cfg = False
@@ -129,11 +154,17 @@ class StaticConfigGenerator(object):
@property
def feature_flags_txt(self):
- return self.__proto_config('feature_flags.txt', feature_flags_pb2.TFeatureFlags, self.__cluster_details.get_service("features"))
+ return self.__proto_config(
+ 'feature_flags.txt', feature_flags_pb2.TFeatureFlags, self.__cluster_details.get_service("features")
+ )
@property
def failure_injection_txt(self):
- return self.__proto_config('failure_injection.txt', config_pb2.TFailureInjectionConfig, self.__cluster_details.get_service("failure_injection_config"))
+ return self.__proto_config(
+ 'failure_injection.txt',
+ config_pb2.TFailureInjectionConfig,
+ self.__cluster_details.get_service("failure_injection_config"),
+ )
@property
def failure_injection_txt_enabled(self):
@@ -145,11 +176,17 @@ class StaticConfigGenerator(object):
@property
def netclassifier_txt(self):
- return self.__proto_config("netclassifier.txt", netclassifier_pb2.TNetClassifierConfig, self.__cluster_details.get_service("netclassifier"))
+ return self.__proto_config(
+ "netclassifier.txt",
+ netclassifier_pb2.TNetClassifierConfig,
+ self.__cluster_details.get_service("netclassifier"),
+ )
@property
def pqcd_txt(self):
- return self.__proto_config("pqcd.txt", pqconfig_pb2.TPQClusterDiscoveryConfig, self.__cluster_details.get_service("pqclusterdiscovery"))
+ return self.__proto_config(
+ "pqcd.txt", pqconfig_pb2.TPQClusterDiscoveryConfig, self.__cluster_details.get_service("pqclusterdiscovery")
+ )
@property
def ic_txt(self):
@@ -161,7 +198,9 @@ class StaticConfigGenerator(object):
@property
def dyn_ns_txt(self):
- return self.__proto_config('dyn_ns.txt', config_pb2.TDynamicNameserviceConfig, self.__cluster_details.dynamicnameservice_config)
+ return self.__proto_config(
+ 'dyn_ns.txt', config_pb2.TDynamicNameserviceConfig, self.__cluster_details.dynamicnameservice_config
+ )
@property
def log_txt(self):
@@ -173,7 +212,9 @@ class StaticConfigGenerator(object):
@property
def vdisks_txt(self):
- return self.__proto_config('vdisks.txt', blobstorage_vdisk_config_pb2.TAllVDiskKinds, self.__cluster_details.vdisk_config)
+ return self.__proto_config(
+ 'vdisks.txt', blobstorage_vdisk_config_pb2.TAllVDiskKinds, self.__cluster_details.vdisk_config
+ )
@property
def sqs_txt(self):
@@ -185,7 +226,9 @@ class StaticConfigGenerator(object):
@property
def rb_txt(self):
- return self.__proto_config('rb.txt', resource_broker_pb2.TResourceBrokerConfig, self.__cluster_details.get_service('resource_broker'))
+ return self.__proto_config(
+ 'rb.txt', resource_broker_pb2.TResourceBrokerConfig, self.__cluster_details.get_service('resource_broker')
+ )
@property
def rb_txt_enabled(self):
@@ -193,7 +236,9 @@ class StaticConfigGenerator(object):
@property
def metering_txt(self):
- return self.__proto_config('metering.txt', config_pb2.TMeteringConfig, self.__cluster_details.get_service("metering"))
+ return self.__proto_config(
+ 'metering.txt', config_pb2.TMeteringConfig, self.__cluster_details.get_service("metering")
+ )
@property
def metering_txt_enabled(self):
@@ -232,9 +277,14 @@ class StaticConfigGenerator(object):
def kikimr_cfg(self):
if self.__is_dynamic_node:
return kikimr_cfg_for_dynamic_node(
- self.__node_broker_port, self._database,
- self.__ic_port, self.__mon_port,
- self.__kikimr_home, self.__sqs_port, self.sqs_txt.EnableSqs, self._enable_cores,
+ self.__node_broker_port,
+ self._database,
+ self.__ic_port,
+ self.__mon_port,
+ self.__kikimr_home,
+ self.__sqs_port,
+ self.sqs_txt.EnableSqs,
+ self._enable_cores,
self.__cluster_details.default_log_level,
mon_address=self.__cluster_details.monitor_address,
cert_params=self.__cluster_details.ic_cert_params,
@@ -242,7 +292,7 @@ class StaticConfigGenerator(object):
rb_txt_enabled=self.rb_txt_enabled,
metering_txt_enabled=self.metering_txt_enabled,
audit_txt_enabled=self.audit_txt_enabled,
- fq_txt_enabled=self.fq_txt_enabled
+ fq_txt_enabled=self.fq_txt_enabled,
)
if self.__cluster_details.use_new_style_kikimr_cfg:
@@ -258,8 +308,11 @@ class StaticConfigGenerator(object):
)
return kikimr_cfg_for_static_node(
- self._database, self.__ic_port,
- self.__mon_port, self.__kikimr_home, self.pq_txt.Enabled,
+ self._database,
+ self.__ic_port,
+ self.__mon_port,
+ self.__kikimr_home,
+ self.pq_txt.Enabled,
self._enable_cores,
self.__cluster_details.default_log_level,
mon_address=self.__cluster_details.monitor_address,
@@ -368,10 +421,7 @@ class StaticConfigGenerator(object):
system_tablets_info[tablet_type] = []
system_tablets_info[tablet_type].append(
- {
- "info": {"tablet_id": tablet["info"]["tablet_id"]},
- "node": tablet["node"]
- }
+ {"info": {"tablet_id": tablet["info"]["tablet_id"]}, "node": tablet["node"]}
)
del normalized_config["bootstrap_config"]["tablet"]
@@ -510,13 +560,9 @@ class StaticConfigGenerator(object):
for channel_id in range(int(number_of_channels)):
channel = tablet.Info.Channels.add(
- Channel=channel_id,
- ChannelErasureName=str(self.__cluster_details.static_erasure)
- )
- channel.History.add(
- FromGeneration=0,
- GroupID=0
+ Channel=channel_id, ChannelErasureName=str(self.__cluster_details.static_erasure)
)
+ channel.History.add(FromGeneration=0, GroupID=0)
@property
def __tablet_types(self):
@@ -548,11 +594,7 @@ class StaticConfigGenerator(object):
for tablet_type, tablet_count in self.__system_tablets:
for index in range(int(tablet_count)):
- self.__add_tablet(
- tablet_type,
- index,
- self.__cluster_details.system_tablets_node_ids
- )
+ self.__add_tablet(tablet_type, index, self.__cluster_details.system_tablets_node_ids)
if self.__cluster_details.shared_cache_memory_limit is not None:
boot_txt = self.__proto_configs["boot.txt"]
@@ -683,13 +725,17 @@ class StaticConfigGenerator(object):
if self.__cluster_details.nw_cache_file_path is not None:
self.__proto_configs['bs.txt'].CacheFilePath = self.__cluster_details.nw_cache_file_path
- def _read_generated_bs_config(self, static_erasure, min_fail_domains, static_pdisk_type, fail_domain_type, bs_format_config):
+ def _read_generated_bs_config(
+ self, static_erasure, min_fail_domains, static_pdisk_type, fail_domain_type, bs_format_config
+ ):
result = config_pb2.TBlobStorageConfig()
with tempfile.NamedTemporaryFile(delete=True) as t_file:
utils.write_proto_to_file(t_file.name, bs_format_config)
- rx_begin, rx_end, dx_begin, dx_end = types.DistinctionLevels[types.FailDomainType.from_string(fail_domain_type)]
+ rx_begin, rx_end, dx_begin, dx_end = types.DistinctionLevels[
+ types.FailDomainType.from_string(fail_domain_type)
+ ]
cmd_base = [
self.__local_binary_path,
@@ -697,25 +743,42 @@ class StaticConfigGenerator(object):
"bs",
"genconfig",
"static",
- "--bs-format-file", t_file.name,
- "--erasure", static_erasure,
- "--avdomain", "1",
- "--faildomains", min_fail_domains,
- "--vdisks", "1",
- "--pdisktype", static_pdisk_type,
+ "--bs-format-file",
+ t_file.name,
+ "--erasure",
+ static_erasure,
+ "--avdomain",
+ "1",
+ "--faildomains",
+ min_fail_domains,
+ "--vdisks",
+ "1",
+ "--pdisktype",
+ static_pdisk_type,
]
try:
- output = subprocess.check_output(cmd_base + [
- "--ring-level-begin", str(rx_begin),
- "--ring-level-end", str(rx_end),
- "--domain-level-begin", str(dx_begin),
- "--domain-level-end", str(dx_end),
- ])
+ output = subprocess.check_output(
+ cmd_base
+ + [
+ "--ring-level-begin",
+ str(rx_begin),
+ "--ring-level-end",
+ str(rx_end),
+ "--domain-level-begin",
+ str(dx_begin),
+ "--domain-level-end",
+ str(dx_end),
+ ]
+ )
except subprocess.CalledProcessError:
- output = subprocess.check_output(cmd_base + [
- "--dx", fail_domain_type,
- ])
+ output = subprocess.check_output(
+ cmd_base
+ + [
+ "--dx",
+ fail_domain_type,
+ ]
+ )
utils.read_message_from_string(output, result)
@@ -745,15 +808,13 @@ class StaticConfigGenerator(object):
profile.Channel.add(
ErasureSpecies=str(self.__cluster_details.static_erasure),
PDiskCategory=chosen_category,
- VDiskCategory=str(types.VDiskCategory.Default)
+ VDiskCategory=str(types.VDiskCategory.Default),
)
for user_profile in self.__cluster_details.tablet_profiles:
-
profile = channels_config.Profile.add()
profile.ProfileId = next(profile_id)
for user_profile_channel in user_profile.channels:
-
params = {
'ErasureSpecies': str(user_profile_channel.erasure),
'PDiskCategory': user_profile_channel.pdisk_type,
@@ -775,9 +836,8 @@ class StaticConfigGenerator(object):
n_to_select_candidate = nodes_count
else:
raise RuntimeError(
- "Unable to configure state storage, n to select %d > length of hosts %d" % (
- n_to_select_candidate, nodes_count
- )
+ "Unable to configure state storage, n to select %d > length of hosts %d"
+ % (n_to_select_candidate, nodes_count)
)
return n_to_select_candidate
@@ -809,9 +869,7 @@ class StaticConfigGenerator(object):
domain_id = domain_description.domain_id
domain_name = domain_description.domain_name
domain = domains_config.Domain.add(
- Name=domain_name,
- DomainId=domain_id,
- PlanResolution=domain_description.plan_resolution
+ Name=domain_name, DomainId=domain_id, PlanResolution=domain_description.plan_resolution
)
domain.SSId.append(domain_id)
domain.HiveUid.append(domain_id)
@@ -824,24 +882,18 @@ class StaticConfigGenerator(object):
domain.SchemeRoot = schemeroot
domain.ExplicitCoordinators.extend(
- [tablet_types.FLAT_TX_COORDINATOR.tablet_id_for(i)
- for i in range(int(domain_description.coordinators))]
+ [tablet_types.FLAT_TX_COORDINATOR.tablet_id_for(i) for i in range(int(domain_description.coordinators))]
)
domain.ExplicitMediators.extend(
- [tablet_types.TX_MEDIATOR.tablet_id_for(i)
- for i in range(int(domain_description.mediators))]
+ [tablet_types.TX_MEDIATOR.tablet_id_for(i) for i in range(int(domain_description.mediators))]
)
domain.ExplicitAllocators.extend(
- [tablet_types.TX_ALLOCATOR.tablet_id_for(i)
- for i in range(int(domain_description.allocators))]
+ [tablet_types.TX_ALLOCATOR.tablet_id_for(i) for i in range(int(domain_description.allocators))]
)
self._configure_statestorages(domains_config, domain_id)
- domains_config.HiveConfig.add(
- HiveUid=domain_id,
- Hive=tablet_types.FLAT_HIVE.tablet_id_for(0)
- )
+ domains_config.HiveConfig.add(HiveUid=domain_id, Hive=tablet_types.FLAT_HIVE.tablet_id_for(0))
for pool_kind in domain_description.storage_pool_kinds.values():
pool_type = domain.StoragePoolTypes.add(Kind=pool_kind.kind)
@@ -868,14 +920,10 @@ class StaticConfigGenerator(object):
if 'type' in pool_kind.filter_properties:
pdisk_type = pool_kind.filter_properties['type']
pdisk_category = int(types.PDiskCategory.from_string(pdisk_type))
- pdisk_filter.Property.add(
- Type=pdisk_category
- )
+ pdisk_filter.Property.add(Type=pdisk_category)
if "SharedWithOs" in pool_kind.filter_properties:
- pdisk_filter.Property.add(
- SharedWithOs=pool_kind.filter_properties["SharedWithOs"]
- )
+ pdisk_filter.Property.add(SharedWithOs=pool_kind.filter_properties["SharedWithOs"])
def _get_base_statestorage(self, domains_cfg, ss):
ssid = ss.get('ssid', None)
@@ -934,8 +982,7 @@ class StaticConfigGenerator(object):
rack_sizes = collections.Counter()
for node in self.__cluster_details.hosts:
rack_sizes[node.rack] += 1
- racks[node.rack].append(
- node)
+ racks[node.rack].append(node)
chosen_racks = []
dc_limit = n_to_select == 9
@@ -985,9 +1032,7 @@ class StaticConfigGenerator(object):
return
if self.__cluster_details.allow_incorrect_state_storage:
- logger.warning(
- "Using unsafe option: "
- "state storage in the cluster is probably broken")
+ logger.warning("Using unsafe option: " "state storage in the cluster is probably broken")
state_storage_cfg.Ring.NToSelect = self.__n_to_select
state_storage_cfg.Ring.Node.extend(self.__cluster_details.state_storage_node_ids)
return
@@ -1050,7 +1095,6 @@ class StaticConfigGenerator(object):
node.Location.Body = int(host.body)
if self.__cluster_details.use_cluster_uuid:
-
accepted_uuids = self.__cluster_details.accepted_cluster_uuids
cluster_uuid = self.__cluster_details.cluster_uuid
cluster_uuid = 'ydb:{}'.format(utils.uuid()) if cluster_uuid is None else cluster_uuid
@@ -1063,7 +1107,8 @@ class StaticConfigGenerator(object):
if self.__cluster_details.sys_preset_name is not None:
utils.read_from_resource(
self.__proto_configs['sys.txt'],
- 'sys', self.__cluster_details.sys_preset_name,
+ 'sys',
+ self.__cluster_details.sys_preset_name,
)
elif self.__cluster_details.use_auto_config:
sys_config = self.__proto_configs['sys.txt']
@@ -1083,7 +1128,12 @@ class StaticConfigGenerator(object):
def __generate_tracing_txt(self):
pb = config_pb2.TAppConfig()
if self.__tracing:
- pb.TracingConfig.Host, pb.TracingConfig.Port, pb.TracingConfig.RootCA, pb.TracingConfig.ServiceName = self.__tracing
+ (
+ pb.TracingConfig.Host,
+ pb.TracingConfig.Port,
+ pb.TracingConfig.RootCA,
+ pb.TracingConfig.ServiceName,
+ ) = self.__tracing
self.__proto_configs['tracing.txt'] = pb
def __generate_sys_txt_advanced(self):
@@ -1102,7 +1152,7 @@ class StaticConfigGenerator(object):
'Threads': 16,
'SpinThreshold': 1,
'Name': "User",
- 'ExecutorUser': 'UserExecutor'
+ 'ExecutorUser': 'UserExecutor',
},
{
'Type': sys_config.TExecutor.EType.Value('BASIC'),
@@ -1123,7 +1173,7 @@ class StaticConfigGenerator(object):
'SpinThreshold': 10,
'Name': "IC",
'TimePerMailboxMicroSecs': 100,
- 'ExecutorUser': 'Interconnect'
+ 'ExecutorUser': 'Interconnect',
},
]
scheduler = {
@@ -1176,5 +1226,6 @@ class StaticConfigGenerator(object):
def dynamic_server_common_args(self):
if self.__cluster_details.use_new_style_kikimr_cfg:
return dynamic_cfg_new_style(self._enable_cores)
- return kikimr_cfg_for_dynamic_slot(self._enable_cores, self._enable_cms_config_cache,
- cert_params=self.__cluster_details.ic_cert_params)
+ return kikimr_cfg_for_dynamic_slot(
+ self._enable_cores, self._enable_cms_config_cache, cert_params=self.__cluster_details.ic_cert_params
+ )
diff --git a/ydb/tools/cfg/templates.py b/ydb/tools/cfg/templates.py
index f88ee6a15c..7f509afe5e 100644
--- a/ydb/tools/cfg/templates.py
+++ b/ydb/tools/cfg/templates.py
@@ -173,21 +173,28 @@ DEFAULT_KIKIMR_MBUS_MAX_MESSAGE_SIZE = 140000000
def local_vars(
- tenant, node_broker_port=None,
- ic_port=19001, mon_port=8765, grpc_port=2135,
- mbus_port=2134,
- kikimr_home='/Berkanavt/kikimr',
- kikimr_binaries_base_path='/Berkanavt/kikimr',
- pq_enable=False, sqs_port=8771, sqs_enable=False,
- enable_cores=False, default_log_level=3, mon_address="",
- cert_params=None,
- rb_txt_enabled=False,
- metering_txt_enabled=False,
- audit_txt_enabled=False,
- yql_txt_enabled=False,
- fq_txt_enabled=False,
- new_style_kikimr_cfg=False,
- mbus_enabled=False,
+ tenant,
+ node_broker_port=None,
+ ic_port=19001,
+ mon_port=8765,
+ grpc_port=2135,
+ mbus_port=2134,
+ kikimr_home='/Berkanavt/kikimr',
+ kikimr_binaries_base_path='/Berkanavt/kikimr',
+ pq_enable=False,
+ sqs_port=8771,
+ sqs_enable=False,
+ enable_cores=False,
+ default_log_level=3,
+ mon_address="",
+ cert_params=None,
+ rb_txt_enabled=False,
+ metering_txt_enabled=False,
+ audit_txt_enabled=False,
+ yql_txt_enabled=False,
+ fq_txt_enabled=False,
+ new_style_kikimr_cfg=False,
+ mbus_enabled=False,
):
cur_vars = []
if enable_cores:
@@ -206,9 +213,7 @@ def local_vars(
if node_broker_port:
cur_vars.append(('kikimr_node_broker_port', node_broker_port))
- cur_vars.append(
- ('kikimr_mon_address', mon_address)
- )
+ cur_vars.append(('kikimr_mon_address', mon_address))
if default_log_level:
cur_vars.extend(
@@ -264,11 +269,7 @@ def local_vars(
cur_vars.append(("kikimr_mbus_infly_bytes", DEFAULT_KIKIMR_MBUS_INFLY_BYTES))
cur_vars.append(("kikimr_mbus_max_message_size", DEFAULT_KIKIMR_MBUS_MAX_MESSAGE_SIZE))
- return '\n'.join(
- [
- '%s="%s"' % (cur_var[0], cur_var[1]) for cur_var in cur_vars
- ]
- )
+ return '\n'.join(['%s="%s"' % (cur_var[0], cur_var[1]) for cur_var in cur_vars])
def tenant_argument(tenant=None):
@@ -280,55 +281,87 @@ def pq_enable_argument(pq_enable=False):
def sqs_arguments(sqs_enable=False):
- return '' if not sqs_enable else '\n'.join(
- [
- 'kikimr_arg="${kikimr_arg}${kikimr_sqs_file:+ --sqs-file ${kikimr_sqs_file}}"',
- 'kikimr_arg="${kikimr_arg}${kikimr_sqs_port:+ --sqs-port ${kikimr_sqs_port}}"'
- ]
+ return (
+ ''
+ if not sqs_enable
+ else '\n'.join(
+ [
+ 'kikimr_arg="${kikimr_arg}${kikimr_sqs_file:+ --sqs-file ${kikimr_sqs_file}}"',
+ 'kikimr_arg="${kikimr_arg}${kikimr_sqs_port:+ --sqs-port ${kikimr_sqs_port}}"',
+ ]
+ )
)
def rb_arguments(rb_txt_enabled=False):
- return [] if not rb_txt_enabled else [
- 'kikimr_arg="${kikimr_arg}${kikimr_rb_file:+ --rb-file ${kikimr_rb_file}}"',
- ]
+ return (
+ []
+ if not rb_txt_enabled
+ else [
+ 'kikimr_arg="${kikimr_arg}${kikimr_rb_file:+ --rb-file ${kikimr_rb_file}}"',
+ ]
+ )
def metering_arguments(metering_txt_enabled=False):
- return [] if not metering_txt_enabled else [
- 'kikimr_arg="${kikimr_arg}${kikimr_metering_file:+ --metering-file ${kikimr_metering_file}}"',
- ]
+ return (
+ []
+ if not metering_txt_enabled
+ else [
+ 'kikimr_arg="${kikimr_arg}${kikimr_metering_file:+ --metering-file ${kikimr_metering_file}}"',
+ ]
+ )
def audit_arguments(audit_txt_enabled=False):
- return [] if not audit_txt_enabled else [
- 'kikimr_arg="${kikimr_arg}${kikimr_audit_file:+ --audit-file ${kikimr_audit_file}}"',
- ]
+ return (
+ []
+ if not audit_txt_enabled
+ else [
+ 'kikimr_arg="${kikimr_arg}${kikimr_audit_file:+ --audit-file ${kikimr_audit_file}}"',
+ ]
+ )
def yql_arguments(yql_txt_enabled=False):
- return [] if not yql_txt_enabled else [
- 'kikimr_arg="${kikimr_arg}${kikimr_yql_file:+ --yql-file ${kikimr_yql_file}}"',
- ]
+ return (
+ []
+ if not yql_txt_enabled
+ else [
+ 'kikimr_arg="${kikimr_arg}${kikimr_yql_file:+ --yql-file ${kikimr_yql_file}}"',
+ ]
+ )
def cms_config_cache_argument_for_dynamic_nodes(enable_cms_config_cache=False):
- return '' if not enable_cms_config_cache else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}_${kikimr_ic_port}/cache/cached_cfg.txt"'
+ return (
+ ''
+ if not enable_cms_config_cache
+ else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}_${kikimr_ic_port}/cache/cached_cfg.txt"'
+ )
def cms_config_cache_argument_for_static_nodes(enable_cms_config_cache=False):
- return '' if not enable_cms_config_cache else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}/cache/cached_cfg.txt"'
+ return (
+ ''
+ if not enable_cms_config_cache
+ else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}/cache/cached_cfg.txt"'
+ )
def mbus_arguments(enable_mbus=False):
- return [] if not enable_mbus else [
- 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ ${kikimr_mbus_enable}}"',
- 'kikimr_arg="${kikimr_arg}${kikimr_mbus_port:+ --mbus-port ${kikimr_mbus_port}}"',
- 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ --mbus-worker-count=4}"',
- 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly:+ --mbus-max-in-flight ${kikimr_mbus_infly}}"',
- 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly_bytes:+ --mbus-max-in-flight-by-size ${kikimr_mbus_infly_bytes}}"',
- 'kikimr_arg="${kikimr_arg}${kikimr_mbus_max_message_size:+ --mbus-max-message-size ${kikimr_mbus_max_message_size}}"',
- ]
+ return (
+ []
+ if not enable_mbus
+ else [
+ 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ ${kikimr_mbus_enable}}"',
+ 'kikimr_arg="${kikimr_arg}${kikimr_mbus_port:+ --mbus-port ${kikimr_mbus_port}}"',
+ 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ --mbus-worker-count=4}"',
+ 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly:+ --mbus-max-in-flight ${kikimr_mbus_infly}}"',
+ 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly_bytes:+ --mbus-max-in-flight-by-size ${kikimr_mbus_infly_bytes}}"',
+ 'kikimr_arg="${kikimr_arg}${kikimr_mbus_max_message_size:+ --mbus-max-message-size ${kikimr_mbus_max_message_size}}"',
+ ]
+ )
def dynamic_cfg_new_style(
@@ -354,47 +387,53 @@ def kikimr_cfg_for_static_node_new_style(
new_style_kikimr_cfg=True,
mbus_enabled=False,
):
-
return "\n".join(
[
local_vars(
- tenant, ic_port=ic_port,
+ tenant,
+ ic_port=ic_port,
mon_address=mon_address,
- mon_port=mon_port, kikimr_home=kikimr_home,
+ mon_port=mon_port,
+ kikimr_home=kikimr_home,
enable_cores=enable_cores,
cert_params=cert_params,
default_log_level=None,
kikimr_binaries_base_path=None,
new_style_kikimr_cfg=new_style_kikimr_cfg,
- mbus_enabled=mbus_enabled
+ mbus_enabled=mbus_enabled,
),
NEW_STYLE_CONFIG,
- ] + mbus_arguments(mbus_enabled)
+ ]
+ + mbus_arguments(mbus_enabled)
)
def kikimr_cfg_for_static_node(
- tenant=None,
- ic_port=19001, mon_port=8765, kikimr_home='/Berkanavt/kikimr',
- pq_enable=False,
- enable_cores=False,
- default_log_level=3,
- kikimr_binaries_base_path='/Berkanavt/kikimr',
- mon_address="",
- cert_params=None,
- enable_cms_config_cache=False,
- rb_txt_enabled=False,
- metering_txt_enabled=False,
- audit_txt_enabled=False,
- yql_txt_enabled=False,
- fq_txt_enabled=False,
- mbus_enabled=False
+ tenant=None,
+ ic_port=19001,
+ mon_port=8765,
+ kikimr_home='/Berkanavt/kikimr',
+ pq_enable=False,
+ enable_cores=False,
+ default_log_level=3,
+ kikimr_binaries_base_path='/Berkanavt/kikimr',
+ mon_address="",
+ cert_params=None,
+ enable_cms_config_cache=False,
+ rb_txt_enabled=False,
+ metering_txt_enabled=False,
+ audit_txt_enabled=False,
+ yql_txt_enabled=False,
+ fq_txt_enabled=False,
+ mbus_enabled=False,
):
return '\n'.join(
[
local_vars(
- tenant, ic_port=ic_port,
- mon_port=mon_port, kikimr_home=kikimr_home,
+ tenant,
+ ic_port=ic_port,
+ mon_port=mon_port,
+ kikimr_home=kikimr_home,
kikimr_binaries_base_path=kikimr_binaries_base_path,
pq_enable=pq_enable,
enable_cores=enable_cores,
@@ -406,7 +445,7 @@ def kikimr_cfg_for_static_node(
audit_txt_enabled=audit_txt_enabled,
yql_txt_enabled=yql_txt_enabled,
fq_txt_enabled=fq_txt_enabled,
- mbus_enabled=mbus_enabled
+ mbus_enabled=mbus_enabled,
),
NODE_ID_LOCAL_VAR,
CUSTOM_CONFIG_INJECTOR,
@@ -415,35 +454,43 @@ def kikimr_cfg_for_static_node(
NODE_ID_ARGUMENT,
tenant_argument(tenant),
pq_enable_argument(pq_enable),
- ] + rb_arguments(rb_txt_enabled) \
- + metering_arguments(metering_txt_enabled) \
- + audit_arguments(audit_txt_enabled) \
- + yql_arguments(yql_txt_enabled) \
- + ([cms_config_cache_argument_for_static_nodes(enable_cms_config_cache)] if enable_cms_config_cache else []) \
+ ]
+ + rb_arguments(rb_txt_enabled)
+ + metering_arguments(metering_txt_enabled)
+ + audit_arguments(audit_txt_enabled)
+ + yql_arguments(yql_txt_enabled)
+ + ([cms_config_cache_argument_for_static_nodes(enable_cms_config_cache)] if enable_cms_config_cache else [])
+ mbus_arguments(mbus_enabled)
)
def kikimr_cfg_for_dynamic_node(
- node_broker_port=2135, tenant=None,
- ic_port=19001, mon_port=8765,
- kikimr_home='/Berkanavt/kikimr', sqs_port=8771, sqs_enable=False,
- enable_cores=False,
- default_log_level=3,
- kikimr_binaries_base_path='/Berkanavt/kikimr',
- mon_address="", cert_params=None,
- enable_cms_config_cache=False,
- rb_txt_enabled=False,
- metering_txt_enabled=False,
- audit_txt_enabled=False,
- yql_txt_enabled=False,
- fq_txt_enabled=False
+ node_broker_port=2135,
+ tenant=None,
+ ic_port=19001,
+ mon_port=8765,
+ kikimr_home='/Berkanavt/kikimr',
+ sqs_port=8771,
+ sqs_enable=False,
+ enable_cores=False,
+ default_log_level=3,
+ kikimr_binaries_base_path='/Berkanavt/kikimr',
+ mon_address="",
+ cert_params=None,
+ enable_cms_config_cache=False,
+ rb_txt_enabled=False,
+ metering_txt_enabled=False,
+ audit_txt_enabled=False,
+ yql_txt_enabled=False,
+ fq_txt_enabled=False,
):
return "\n".join(
[
local_vars(
- tenant, node_broker_port=node_broker_port,
- ic_port=ic_port, mon_port=mon_port,
+ tenant,
+ node_broker_port=node_broker_port,
+ ic_port=ic_port,
+ mon_port=mon_port,
kikimr_home=kikimr_home,
kikimr_binaries_base_path=kikimr_binaries_base_path,
sqs_port=sqs_port,
@@ -464,9 +511,10 @@ def kikimr_cfg_for_dynamic_node(
NODE_BROKER_ARGUMENT,
tenant_argument(tenant),
sqs_arguments(sqs_enable),
- ] + rb_arguments(rb_txt_enabled) \
- + metering_arguments(metering_txt_enabled) \
- + audit_arguments(audit_txt_enabled) \
+ ]
+ + rb_arguments(rb_txt_enabled)
+ + metering_arguments(metering_txt_enabled)
+ + audit_arguments(audit_txt_enabled)
+ ([cms_config_cache_argument_for_dynamic_nodes(enable_cms_config_cache)] if enable_cms_config_cache else [])
)
@@ -484,11 +532,7 @@ def expected_vars(**kwargs):
kikimr_patched.extend(BASE_VARS)
- return '\n'.join(
- [
- '%s="%s"' % (var[0], var[1]) for var in kikimr_patched if var[0] not in exclude_options
- ]
- )
+ return '\n'.join(['%s="%s"' % (var[0], var[1]) for var in kikimr_patched if var[0] not in exclude_options])
def kikimr_cfg_for_dynamic_slot(
@@ -515,9 +559,11 @@ def kikimr_cfg_for_dynamic_slot(
mon_port=None,
home=None,
syslog_service_tag=None,
- exclude_options=['kikimr_system_file']
+ exclude_options=['kikimr_system_file'],
),
- ] + ['kikimr_%s=%s' % (key, value) for key, value in zip(['ca', 'cert', 'key'], cert_params or []) if value] + [
+ ]
+ + ['kikimr_%s=%s' % (key, value) for key, value in zip(['ca', 'cert', 'key'], cert_params or []) if value]
+ + [
CUSTOM_CONFIG_INJECTOR,
CUSTOM_SYS_CONFIG_INJECTOR,
# arguments
@@ -525,9 +571,10 @@ def kikimr_cfg_for_dynamic_slot(
NODE_BROKER_ARGUMENT,
SYS_LOG_SERVICE_TAG,
tenant_argument(True),
- ] + rb_arguments(rb_txt_enabled) \
- + metering_arguments(metering_txt_enabled) \
- + audit_arguments(audit_txt_enabled) \
- + yql_arguments(yql_txt_enabled) \
+ ]
+ + rb_arguments(rb_txt_enabled)
+ + metering_arguments(metering_txt_enabled)
+ + audit_arguments(audit_txt_enabled)
+ + yql_arguments(yql_txt_enabled)
+ ([cms_config_cache_argument_for_dynamic_nodes(enable_cms_config_cache)] if enable_cms_config_cache else [])
)
diff --git a/ydb/tools/cfg/types.py b/ydb/tools/cfg/types.py
index 773f9cc70d..c401e7069d 100644
--- a/ydb/tools/cfg/types.py
+++ b/ydb/tools/cfg/types.py
@@ -24,14 +24,14 @@ def py3_ensure_bytes(value):
@enum.unique
class LogLevels(enum.IntEnum):
- EMERG = 0,
- ALERT = 1,
- CRIT = 2,
- ERROR = 3,
- WARN = 4,
- NOTICE = 5,
- INFO = 6,
- DEBUG = 7,
+ EMERG = (0,)
+ ALERT = (1,)
+ CRIT = (2,)
+ ERROR = (3,)
+ WARN = (4,)
+ NOTICE = (5,)
+ INFO = (6,)
+ DEBUG = (7,)
TRACE = 8
@@ -48,7 +48,7 @@ class TabletTypes(enum.Enum):
FLAT_TX_COORDINATOR = _tablet_type(13, 0x800001, service_name='TX_COORDINATOR')
FLAT_HIVE = _tablet_type(14, 0xA001, is_unique=True, service_name='HIVE')
FLAT_BS_CONTROLLER = _tablet_type(15, 0x1001, is_unique=True, service_name='BS_CONTROLLER')
- FLAT_SCHEMESHARD = _tablet_type(16, 0x1000008587a0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD')
+ FLAT_SCHEMESHARD = _tablet_type(16, 0x1000008587A0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD')
TX_ALLOCATOR = _tablet_type(23, 0x820001, service_name='TX_ALLOCATOR')
FLAT_TX_PROXY = _tablet_type(17, 0x820001, service_name='TX_PROXY')
FLAT_DATASHARD = _tablet_type(18, 999, service_name='DATASHARD')
@@ -110,7 +110,7 @@ class TabletTypesFixed(enum.Enum):
FLAT_TX_COORDINATOR = _tablet_type(13, 0x800001, service_name='TX_COORDINATOR')
FLAT_HIVE = _tablet_type(14, 0xA001, is_unique=True, service_name='HIVE')
FLAT_BS_CONTROLLER = _tablet_type(15, 0x1001, is_unique=True, service_name='BS_CONTROLLER')
- FLAT_SCHEMESHARD = _tablet_type(16, 0x8587a0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD')
+ FLAT_SCHEMESHARD = _tablet_type(16, 0x8587A0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD')
TX_ALLOCATOR = _tablet_type(23, 0x820001, service_name='TX_ALLOCATOR')
FLAT_TX_PROXY = _tablet_type(17, 0x820001, service_name='TX_PROXY')
FLAT_DATASHARD = _tablet_type(18, 999, service_name='DATASHARD')
diff --git a/ydb/tools/cfg/utils.py b/ydb/tools/cfg/utils.py
index 98a285dbad..94c0533459 100644
--- a/ydb/tools/cfg/utils.py
+++ b/ydb/tools/cfg/utils.py
@@ -43,12 +43,7 @@ def write_to_file(file_path, value):
def write_proto_to_file(file_path, proto):
- write_to_file(
- file_path,
- message_to_string(
- proto
- )
- )
+ write_to_file(file_path, message_to_string(proto))
def read_message_from_string(value, message):
@@ -70,8 +65,7 @@ def get_resource_data(*args):
def read_from_resource(message, *args):
for key, value in resource.iteritems(get_resource_prefix(*args)):
- return read_message_from_string(
- value, message)
+ return read_message_from_string(value, message)
raise RuntimeError()
@@ -83,12 +77,7 @@ def get_resources_list(prefix=''):
def capitalize_name(initial_name):
- return ''.join(
- map(
- lambda name_part: name_part.capitalize(),
- initial_name.split('_')
- )
- )
+ return ''.join(map(lambda name_part: name_part.capitalize(), initial_name.split('_')))
def to_lower(s):
@@ -97,11 +86,7 @@ def to_lower(s):
def get_canonical_name(name):
chars = set(string.punctuation)
- return to_lower(
- "".join(
- filter(lambda x: x not in chars, name)
- )
- )
+ return to_lower("".join(filter(lambda x: x not in chars, name)))
def apply_config_changes(target, changes, fix_names=None):
@@ -122,7 +107,10 @@ def apply_config_changes(target, changes, fix_names=None):
if fixed_change_name is None:
fixed_change_name = in_proto_field_names.get(get_canonical_name(change_name))
- assert fixed_change_name is not None, "Cannot find suitable field %s, proto descriptor: %s" % (change_name, target.DESCRIPTOR.full_name)
+ assert fixed_change_name is not None, "Cannot find suitable field %s, proto descriptor: %s" % (
+ change_name,
+ target.DESCRIPTOR.full_name,
+ )
if isinstance(change_value, list):
for item in change_value:
@@ -150,7 +138,8 @@ def apply_config_changes(target, changes, fix_names=None):
setattr(target, fixed_change_name, change_value)
except Exception as e:
raise RuntimeError(
- "Cannot apply config change: change_name: %s, change_value %s. Reason: %s" % (
+ "Cannot apply config change: change_name: %s, change_value %s. Reason: %s"
+ % (
change_name,
str(change_value),
str(e),
diff --git a/ydb/tools/cfg/validation.py b/ydb/tools/cfg/validation.py
index 2758d73814..c764e2b372 100644
--- a/ydb/tools/cfg/validation.py
+++ b/ydb/tools/cfg/validation.py
@@ -5,15 +5,12 @@ import copy
import jsonschema
from ydb.tools.cfg import utils
-from ydb.tools.cfg.types import (Erasure, FailDomainType, LogLevels, NodeType,
- PDiskCategory)
+from ydb.tools.cfg.types import Erasure, FailDomainType, LogLevels, NodeType, PDiskCategory
KQP_SCHEMA = {
'type': 'object',
'properties': {
- 'enable': {
- 'type': 'boolean'
- },
+ 'enable': {'type': 'boolean'},
},
}
@@ -45,12 +42,12 @@ PROFILES = {
'type': 'string',
},
},
- "additionalProperties": False
+ "additionalProperties": False,
},
- "minItems": 3
+ "minItems": 3,
}
- }
- }
+ },
+ },
}
@@ -90,7 +87,7 @@ EXECUTOR_SCHEMA = {
'type': 'integer',
'min': 1,
},
- }
+ },
}
SYS_SCHEMA = {
@@ -109,15 +106,15 @@ SYS_SCHEMA = {
'io': copy.deepcopy(EXECUTOR_SCHEMA),
'ic': copy.deepcopy(EXECUTOR_SCHEMA),
},
- 'additionalProperties': False
+ 'additionalProperties': False,
},
'scheduler': {
'type': 'object',
'properties': {
'resolution': {'type': 'integer'},
'spin_threshold': {'type': 'integer'},
- 'progress_threshold': {'type': 'integer'}
- }
+ 'progress_threshold': {'type': 'integer'},
+ },
},
'use_auto_config': {'type': 'boolean'},
'cpu_count': {'type': 'integer'},
@@ -126,7 +123,7 @@ SYS_SCHEMA = {
'enum': NodeType.all_node_type_names(),
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
}
TRACING_SCHEMA = dict(
@@ -143,15 +140,13 @@ TRACING_SCHEMA = dict(
'root_ca',
'service_name',
],
- additionalProperties=False
+ additionalProperties=False,
)
FAILURE_INJECTION_CONFIG_SCHEMA = {
"type": "object",
- "properties": {
- 'approximate_termination_interval': dict(type='integer')
- },
- "additionalProperties": False
+ "properties": {'approximate_termination_interval': dict(type='integer')},
+ "additionalProperties": False,
}
DRIVE_SCHEMA = {
@@ -161,30 +156,18 @@ DRIVE_SCHEMA = {
'path': dict(type='string', minLength=1),
'shared_with_os': dict(type='boolean'),
'expected_slot_count': dict(type='integer'),
- 'kind': dict(type='integer')
+ 'kind': dict(type='integer'),
},
- "required": [
- "type",
- "path"
- ],
- "additionalProperties": False
+ "required": ["type", "path"],
+ "additionalProperties": False,
}
HOST_SCHEMA = {
"type": "object",
"properties": {
- "name": {
- "type": "string",
- "minLength": 1
- },
- "drives": {
- "type": "array",
- "items": copy.deepcopy(DRIVE_SCHEMA)
- },
- 'host_config_id': {
- 'type': "integer",
- 'minLength': 1
- },
+ "name": {"type": "string", "minLength": 1},
+ "drives": {"type": "array", "items": copy.deepcopy(DRIVE_SCHEMA)},
+ 'host_config_id': {'type': "integer", 'minLength': 1},
"ic_port": {
"type": "integer",
},
@@ -197,29 +180,22 @@ HOST_SCHEMA = {
LOG_SCHEMA = {
'type': "object",
"properties": {
- "default": {
- "type": "integer",
- "min": int(min(LogLevels)),
- "max": int(max(LogLevels))
- },
+ "default": {"type": "integer", "min": int(min(LogLevels)), "max": int(max(LogLevels))},
"entries": {
"type": "array",
"items": {
"type": "object",
"properties": {
- "name": {
- "type": "string",
- "minLength": 1
- },
+ "name": {"type": "string", "minLength": 1},
"level": {
"type": "integer",
"min": int(min(LogLevels)),
"max": int(max(LogLevels)),
- }
- }
- }
- }
- }
+ },
+ },
+ },
+ },
+ },
}
STORAGE_POOL = {
@@ -230,23 +206,13 @@ STORAGE_POOL = {
'minLength': 1,
'enum': Erasure.all_erasure_type_names(),
},
- 'generation': {
- 'type': 'integer',
- 'min': 0
- },
- 'encryption_mode': {
- 'type': 'integer',
- 'min': 0,
- 'max': 1
- },
+ 'generation': {'type': 'integer', 'min': 0},
+ 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1},
'num_groups': {
'type': 'integer',
'min': 1,
},
- 'storage_pool_id': {
- 'type': 'integer',
- 'min': 1
- },
+ 'storage_pool_id': {'type': 'integer', 'min': 1},
"fail_domain_type": {
"type": "string",
"minLength": 1,
@@ -273,7 +239,7 @@ STORAGE_POOL = {
'kind': dict(type='integer'),
},
'required': ['type'],
- 'additionalProperties': False
+ 'additionalProperties': False,
},
'vdisk_kind': {
'type': 'string',
@@ -300,11 +266,7 @@ STORAGE_POOL_KIND = {
"minLength": 1,
"enum": Erasure.all_erasure_type_names(),
},
- 'encryption_mode': {
- 'type': 'integer',
- 'min': 0,
- 'max': 1
- },
+ 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1},
"filter_properties": {
"type": "object",
"properties": {
@@ -318,7 +280,7 @@ STORAGE_POOL_KIND = {
'kind': dict(type='integer'),
},
"required": ["type"],
- "additionalProperties": False
+ "additionalProperties": False,
},
"fail_domain_type": {
"type": "string",
@@ -330,11 +292,7 @@ STORAGE_POOL_KIND = {
'minLength': 1,
},
},
- "required": [
- "kind",
- "erasure",
- "filter_properties"
- ],
+ "required": ["kind", "erasure", "filter_properties"],
}
STORAGE_POOL_INSTANCE = {
@@ -351,7 +309,7 @@ STORAGE_POOL_INSTANCE = {
'generation': {
'type': 'integer',
'min': 0,
- }
+ },
},
'required': [
'kind',
@@ -363,12 +321,8 @@ STORAGE_POOL_INSTANCE = {
SHARED_CACHE_SCHEMA = {
'type': 'object',
- 'properties': {
- 'memory_limit': {
- 'type': 'integer'
- }
- },
- 'additionalProperties': False
+ 'properties': {'memory_limit': {'type': 'integer'}},
+ 'additionalProperties': False,
}
TENANT_SCHEMA = {
@@ -386,12 +340,9 @@ TENANT_SCHEMA = {
'count': {
'type': 'integer',
},
- 'kind': {
- 'type': 'string',
- 'minLength': 1
- }
- }
- }
+ 'kind': {'type': 'string', 'minLength': 1},
+ },
+ },
},
'compute_units': {
'type': 'array',
@@ -406,56 +357,35 @@ TENANT_SCHEMA = {
'type': 'string',
'minLength': 1,
},
- 'zone': {
- 'type': 'string',
- 'minLength': 1
- },
+ 'zone': {'type': 'string', 'minLength': 1},
'required': ['count', 'kind', 'zone'],
- 'additionalProperties': False
- }
- }
+ 'additionalProperties': False,
+ },
+ },
},
'overridden_configs': {
'type': 'object',
- }
- }
+ },
+ },
}
DOMAIN_SCHEMA = {
'type': 'object',
'properties': {
- 'domain_name': {
- 'type': 'string',
- 'minLength': 1
- },
+ 'domain_name': {'type': 'string', 'minLength': 1},
'plan_resolution': {
'type': 'integer',
'min': 1,
},
- 'domain_id': {
- 'type': 'integer',
- 'min': 1
- },
+ 'domain_id': {'type': 'integer', 'min': 1},
'mediators': {
'type': 'integer',
'min': 1,
},
- 'coordinators': {
- 'type': 'integer',
- 'min': 1
- },
- 'allocators': {
- 'type': 'integer',
- 'min': 1
- },
- 'storage_pool_kinds': {
- 'type': 'array',
- 'items': copy.deepcopy(STORAGE_POOL_KIND)
- },
- 'storage_pools': {
- 'type': 'array',
- 'items': copy.deepcopy(STORAGE_POOL_INSTANCE)
- },
+ 'coordinators': {'type': 'integer', 'min': 1},
+ 'allocators': {'type': 'integer', 'min': 1},
+ 'storage_pool_kinds': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_KIND)},
+ 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_INSTANCE)},
'databases': {
'type': 'array',
'items': copy.deepcopy(TENANT_SCHEMA),
@@ -463,21 +393,18 @@ DOMAIN_SCHEMA = {
'config_cookie': {
'type': 'string',
},
- 'dynamic_slots': {
- 'type': 'integer',
- 'min': 1
- },
+ 'dynamic_slots': {'type': 'integer', 'min': 1},
'bind_slots_to_numa_nodes': {'type': 'boolean'},
'console_initializers': {
'type': 'array',
'items': {
'type': 'string',
'enum': utils.get_resources_list('resources/console_initializers/'),
- }
+ },
},
},
'required': ['domain_name'],
- 'additionalProperties': False
+ 'additionalProperties': False,
}
NBS_SCHEMA = {
@@ -489,11 +416,7 @@ NBS_SCHEMA = {
'log': copy.deepcopy(LOG_SCHEMA),
'domain': {'type': 'string'},
'subdomain': {'type': 'string'},
- 'storage': {
- 'type': 'object',
- 'properties': {
- }
- },
+ 'storage': {'type': 'object', 'properties': {}},
'disk_registry_proxy': {
'type': 'object',
},
@@ -522,47 +445,25 @@ NBS_SCHEMA = {
},
},
},
- }
- }
- },
- 'discovery': {
- 'type': 'object'
- },
- 'ydbstats': {
- 'type': 'object'
- },
- 'auth': {
- 'type': 'object'
- },
- 'names': {
- 'type': 'object'
+ },
+ },
},
+ 'discovery': {'type': 'object'},
+ 'ydbstats': {'type': 'object'},
+ 'auth': {'type': 'object'},
+ 'names': {'type': 'object'},
'client': {
'type': 'object',
'properties': {
- 'auth_config': {
- 'type': 'object'
- },
- 'client_config': {
- 'type': 'object'
- },
- 'log_config': {
- 'type': 'object'
- },
- 'monitoring_config': {
- 'type': 'object'
- },
- 'throttling_enabled': {
- 'type': 'boolean'
- },
- 'throttling_enabled_s_s_d': {
- 'type': 'boolean'
- },
- 'throttling_disabled_s_s_d_nonrepl': {
- 'type': 'boolean'
- },
+ 'auth_config': {'type': 'object'},
+ 'client_config': {'type': 'object'},
+ 'log_config': {'type': 'object'},
+ 'monitoring_config': {'type': 'object'},
+ 'throttling_enabled': {'type': 'boolean'},
+ 'throttling_enabled_s_s_d': {'type': 'boolean'},
+ 'throttling_disabled_s_s_d_nonrepl': {'type': 'boolean'},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
},
'http_proxy': {
'type': 'object',
@@ -603,7 +504,7 @@ NBS_SCHEMA = {
'type': 'boolean',
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
},
'breakpad': {
'type': 'object',
@@ -612,7 +513,7 @@ NBS_SCHEMA = {
'type': 'boolean',
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
},
'breakpad_sender': {
'type': 'object',
@@ -624,46 +525,23 @@ NBS_SCHEMA = {
'type': 'string',
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
},
'logbroker': {
'type': 'object',
'properties': {
- 'address': {
- 'type': 'string'
- },
- 'port': {
- 'type': 'integer'
- },
- 'database': {
- 'type': 'string'
- },
- 'use_logbroker_c_d_s': {
- 'type': 'boolean'
- },
- 'ca_cert_filename': {
- 'type': 'string'
- },
- 'topic': {
- 'type': 'string'
- },
- 'source_id': {
- 'type': 'string'
- },
- 'metadata_server_address': {
- 'type': 'string'
- }
+ 'address': {'type': 'string'},
+ 'port': {'type': 'integer'},
+ 'database': {'type': 'string'},
+ 'use_logbroker_c_d_s': {'type': 'boolean'},
+ 'ca_cert_filename': {'type': 'string'},
+ 'topic': {'type': 'string'},
+ 'source_id': {'type': 'string'},
+ 'metadata_server_address': {'type': 'string'},
},
- 'additionalProperties': False
- },
- 'notify': {
- 'type': 'object',
- 'properties': {
- 'endpoint': {
- 'type': 'string'
- }
- }
+ 'additionalProperties': False,
},
+ 'notify': {'type': 'object', 'properties': {'endpoint': {'type': 'string'}}},
"iam": {
'type': 'object',
},
@@ -691,9 +569,9 @@ NBS_SCHEMA = {
'type': 'array',
'items': {
'type': 'string',
- }
+ },
}
- }
+ },
},
'Blacklist': {
'type': 'object',
@@ -702,18 +580,18 @@ NBS_SCHEMA = {
'type': 'array',
'items': {
'type': 'string',
- }
+ },
}
- }
+ },
},
},
},
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
}
NFS_SCHEMA = {
@@ -725,18 +603,10 @@ NFS_SCHEMA = {
'log': copy.deepcopy(LOG_SCHEMA),
'domain': {'type': 'string'},
'subdomain': {'type': 'string'},
- 'names': {
- 'type': 'object'
- },
- 'storage': {
- 'type': 'object'
- },
- 'diagnostics': {
- 'type': 'object'
- },
- 'auth': {
- 'type': 'object'
- },
+ 'names': {'type': 'object'},
+ 'storage': {'type': 'object'},
+ 'diagnostics': {'type': 'object'},
+ 'auth': {'type': 'object'},
'server': {
'type': 'object',
'properties': {
@@ -745,8 +615,8 @@ NFS_SCHEMA = {
},
'port': {
'type': 'integer',
- }
- }
+ },
+ },
},
'vhost': {
'type': 'object',
@@ -756,8 +626,8 @@ NFS_SCHEMA = {
},
'port': {
'type': 'integer',
- }
- }
+ },
+ },
},
'http_proxy': {
'type': 'object',
@@ -772,10 +642,10 @@ NFS_SCHEMA = {
'type': 'integer',
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
},
},
- 'additionalProperties': False
+ 'additionalProperties': False,
}
SQS_SCHEMA = {
@@ -787,8 +657,8 @@ SQS_SCHEMA = {
'subdomain': {'type': 'string'},
'http_server': {
'type': 'object',
- }
- }
+ },
+ },
}
SOLOMON_SCHEMA = {
@@ -805,10 +675,10 @@ SOLOMON_SCHEMA = {
'name': {'type': 'string'},
'channels_profile': {'type': 'integer'},
'partitions': {'type': 'integer'},
- }
- }
- }
- }
+ },
+ },
+ },
+ },
}
CMS_LIMITS_SCHEMA = {
@@ -816,7 +686,7 @@ CMS_LIMITS_SCHEMA = {
'properties': {
'disabled_nodes_limit': {'type': 'integer'},
'disabled_nodes_ratio_limit': {'type': 'integer'},
- }
+ },
}
CMS_SCHEMA = {
@@ -837,9 +707,9 @@ CMS_SCHEMA = {
'broken_timeout_min': {'type': 'integer'},
'broken_prep_timeout_min': {'type': 'integer'},
'faulty_prep_timeout_min': {'type': 'integer'},
- }
- }
- }
+ },
+ },
+ },
}
STATE_STORAGE = {
@@ -847,21 +717,11 @@ STATE_STORAGE = {
'properties': {
'allow_incorrect': {'type': 'boolean'},
'node_ids': {'type': 'array', 'items': {'type': 'integer'}},
- 'node_set': {
- 'type': 'array',
- 'items': {
- 'type': 'string'
- }
- }
- }
+ 'node_set': {'type': 'array', 'items': {'type': 'string'}},
+ },
}
-METERING_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'metering_file': {'type': 'string'}
- }
-}
+METERING_SCHEMA = {'type': 'object', 'properties': {'metering_file': {'type': 'string'}}}
YQL_SCHEMA = {
'type': 'object',
@@ -882,8 +742,8 @@ YQL_SCHEMA = {
'impersonation_service_endpoint': {'type': 'string'},
'secure_task_service': {'type': 'boolean'},
'secure_impersonation_service': {'type': 'boolean'},
- 'hmac_secret_file': {'type': 'string'}
- }
+ 'hmac_secret_file': {'type': 'string'},
+ },
}
YQ_SCHEMA = {
@@ -904,7 +764,7 @@ YQ_SCHEMA = {
'use_local_metadata_service': {'type': 'boolean'},
'enable_forward_analytics': {'type': 'boolean'},
'enable_permissions': {'type': 'boolean'},
- }
+ },
},
'analytics': {
'type': 'object',
@@ -926,8 +786,8 @@ YQ_SCHEMA = {
'impersonation_service_endpoint': {'type': 'string'},
'secure_task_service': {'type': 'boolean'},
'secure_impersonation_service': {'type': 'boolean'},
- 'hmac_secret_file': {'type': 'string'}
- }
+ 'hmac_secret_file': {'type': 'string'},
+ },
},
'streaming': {
'type': 'object',
@@ -947,27 +807,24 @@ YQ_SCHEMA = {
'properties': {
'enabled': {'type': 'boolean'},
'period_millis': {'type': 'integer'},
- 'max_in_flight': {'type': 'integer'}
- }
- }
- }
+ 'max_in_flight': {'type': 'integer'},
+ },
+ },
+ },
},
'token_accessor': {
'type': 'object',
- 'properties': {
- 'endpoint': {'type': 'string'},
- 'use_ssl': {'type': 'boolean'}
- }
+ 'properties': {'endpoint': {'type': 'string'}, 'use_ssl': {'type': 'boolean'}},
},
'folder_service': {
'type': 'object',
'properties': {
'enable': {'type': 'boolean'},
'endpoint': {'type': 'string'},
- 'path_to_root_ca': {'type': 'string'}
- }
- }
- }
+ 'path_to_root_ca': {'type': 'string'},
+ },
+ },
+ },
}
TEMPLATE_SCHEMA = {
@@ -988,21 +845,19 @@ TEMPLATE_SCHEMA = {
'items': {
'type': 'object',
'properties': {
- 'generation': {
- 'type': 'integer', 'min': 0
- },
+ 'generation': {'type': 'integer', 'min': 0},
'drives': {
'type': 'array',
'items': copy.deepcopy(DRIVE_SCHEMA),
- }
- }
- }
+ },
+ },
+ },
},
'static_bs_group_hosts': {
'type': 'array',
'items': {
'type': 'string',
- }
+ },
},
'use_cluster_uuid': {
'type': 'boolean',
@@ -1014,32 +869,16 @@ TEMPLATE_SCHEMA = {
'type': 'string',
'minLength': 1,
},
- 'accepted_cluster_uuids': {
- 'type': 'array',
- 'items': {
- 'type': 'string',
- 'minLength': 1
- }
- },
+ 'accepted_cluster_uuids': {'type': 'array', 'items': {'type': 'string', 'minLength': 1}},
'security_settings': {
'type': 'object',
'properties': {
'enforce_user_token_requirement': {
'type': 'boolean',
},
- 'monitoring_allowed_sids': {
- 'type': 'array',
- 'items': {
- 'type': 'string'
- }
- },
- 'administration_allowed_sids': {
- 'type': 'array',
- 'items': {
- 'type': 'string'
- }
- },
- }
+ 'monitoring_allowed_sids': {'type': 'array', 'items': {'type': 'string'}},
+ 'administration_allowed_sids': {'type': 'array', 'items': {'type': 'string'}},
+ },
},
"static_erasure": {
"type": "string",
@@ -1067,24 +906,15 @@ TEMPLATE_SCHEMA = {
'minItems': 1,
'checkNameServiceDuplicates': True,
},
- 'auth': {
- 'type': 'object'
- },
+ 'auth': {'type': 'object'},
'log': copy.deepcopy(LOG_SCHEMA),
'grpc': {'type': 'object'},
'kqp': copy.deepcopy(KQP_SCHEMA),
'ic': {'type': 'object'},
'pq': {'type': 'object'},
- 'storage_pools': {
- 'type': 'array',
- 'items': copy.deepcopy(STORAGE_POOL)
- },
+ 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL)},
'profiles': copy.deepcopy(PROFILES),
- 'domains': {
- 'type': 'array',
- 'items': copy.deepcopy(DOMAIN_SCHEMA),
- 'minItems': 1
- },
+ 'domains': {'type': 'array', 'items': copy.deepcopy(DOMAIN_SCHEMA), 'minItems': 1},
'nbs': copy.deepcopy(NBS_SCHEMA),
'nbs_control': copy.deepcopy(NBS_SCHEMA),
'nfs': copy.deepcopy(NFS_SCHEMA),
@@ -1100,43 +930,28 @@ TEMPLATE_SCHEMA = {
'resource_broker': {'type': 'object'},
'state_storages': {
'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {},
- 'additionalProperties': True
- }
+ 'items': {'type': 'object', 'properties': {}, 'additionalProperties': True},
},
'metering': copy.deepcopy(METERING_SCHEMA),
'yql_analytics': copy.deepcopy(YQL_SCHEMA),
- 'yq': copy.deepcopy(YQ_SCHEMA)
+ 'yq': copy.deepcopy(YQ_SCHEMA),
},
- "required": [
- "static_erasure",
- "hosts"
- ],
+ "required": ["static_erasure", "hosts"],
}
def _host_and_ic_port(host):
- return "%s:%s" % (
- host['name'], str(
- host.get('ic_port', 19001)
- )
- )
+ return "%s:%s" % (host['name'], str(host.get('ic_port', 19001)))
def checkNameServiceDuplicates(validator, allow_duplicates, instance, schema):
- names = collections.Counter(
- [
- _host_and_ic_port(host)
- for host in instance
- ]
- )
+ names = collections.Counter([_host_and_ic_port(host) for host in instance])
for name, count in names.items():
if count > 1:
yield jsonschema.ValidationError(
- "Names of items contains non-unique elements %r: %s. " % (
+ "Names of items contains non-unique elements %r: %s. "
+ % (
instance,
name,
)
@@ -1145,20 +960,17 @@ def checkNameServiceDuplicates(validator, allow_duplicates, instance, schema):
_Validator = jsonschema.Draft4Validator
_Validator = jsonschema.validators.extend(
- _Validator, {
+ _Validator,
+ {
'checkNameServiceDuplicates': checkNameServiceDuplicates,
- }
+ },
)
class Validator(_Validator):
-
def __init__(self, schema):
format_checker = jsonschema.FormatChecker()
- super(Validator, self).__init__(
- schema,
- format_checker=format_checker
- )
+ super(Validator, self).__init__(schema, format_checker=format_checker)
def validate(template):
diff --git a/ydb/tools/cfg/walle/__init__.py b/ydb/tools/cfg/walle/__init__.py
index 58cd44787a..df7d52021c 100644
--- a/ydb/tools/cfg/walle/__init__.py
+++ b/ydb/tools/cfg/walle/__init__.py
@@ -2,6 +2,4 @@
# -*- coding: utf-8 -*-
from .walle import HostsFetcherInterface, NopHostsFetcher # noqa
-__all__ = (
- 'Walle',
-)
+__all__ = ('Walle',)
diff --git a/ydb/tools/cfg/walle/walle.py b/ydb/tools/cfg/walle/walle.py
index 82a5cb70e3..92792aaf6e 100644
--- a/ydb/tools/cfg/walle/walle.py
+++ b/ydb/tools/cfg/walle/walle.py
@@ -4,7 +4,7 @@ import zlib
from abc import ABCMeta, abstractmethod
-class HostsFetcherInterface():
+class HostsFetcherInterface:
__metaclass__ = ABCMeta
@abstractmethod