diff options
author | shmel1k <shmel1k@ydb.tech> | 2023-11-28 12:03:45 +0300 |
---|---|---|
committer | shmel1k <shmel1k@ydb.tech> | 2023-11-28 13:33:54 +0300 |
commit | 781028b5c8b17b25dae1f2bdd8f1c392e2f91e79 (patch) | |
tree | b2a4f59a7168b814ac6bbe70646fc83882f4514a | |
parent | 31cf81679842e3335cbcfeae3a52bd4cbb829c9a (diff) | |
download | ydb-781028b5c8b17b25dae1f2bdd8f1c392e2f91e79.tar.gz |
improve styling for ydb_configure
-rw-r--r-- | ydb/tools/cfg/base.py | 238 | ||||
-rw-r--r-- | ydb/tools/cfg/bin/__main__.py | 53 | ||||
-rw-r--r-- | ydb/tools/cfg/configurator_setup.py | 87 | ||||
-rw-r--r-- | ydb/tools/cfg/dynamic.py | 102 | ||||
-rw-r--r-- | ydb/tools/cfg/static.py | 267 | ||||
-rw-r--r-- | ydb/tools/cfg/templates.py | 261 | ||||
-rw-r--r-- | ydb/tools/cfg/types.py | 20 | ||||
-rw-r--r-- | ydb/tools/cfg/utils.py | 31 | ||||
-rw-r--r-- | ydb/tools/cfg/validation.py | 486 | ||||
-rw-r--r-- | ydb/tools/cfg/walle/__init__.py | 4 | ||||
-rw-r--r-- | ydb/tools/cfg/walle/walle.py | 2 |
11 files changed, 665 insertions, 886 deletions
diff --git a/ydb/tools/cfg/base.py b/ydb/tools/cfg/base.py index ae96212e59..f139cfba32 100644 --- a/ydb/tools/cfg/base.py +++ b/ydb/tools/cfg/base.py @@ -26,7 +26,11 @@ DEFAULT_BOX_ID = 1 VDISKS_DEFAULT_CONFIG = { 'vdisk_kinds': [ {'kind': 'Log', 'base_kind': 'Default', 'config': {'fresh_use_dreg': True, 'allow_keep_flags': False}}, - {'kind': 'LocalMode', 'base_kind': 'Default', 'config': {'hull_comp_level0_max_ssts_at_once': 2, 'hull_comp_sorted_parts_num': 1}}, + { + 'kind': 'LocalMode', + 'base_kind': 'Default', + 'config': {'hull_comp_level0_max_ssts_at_once': 2, 'hull_comp_sorted_parts_num': 1}, + }, ] } @@ -42,9 +46,7 @@ GRPC_DEFAULT_CONFIG = { 'keep_alive_probe_interval_sec': 10, } -DYNAMIC_NAME_SERVICE = { - 'max_static_node_id': 50000 -} +DYNAMIC_NAME_SERVICE = {'max_static_node_id': 50000} IC_DEFAULT_CONFIG = { 'start_tcp': True, @@ -67,19 +69,22 @@ class KiKiMRDrive(object): def __eq__(self, other): return ( - self.type == other.type and - self.path == other.path and - self.shared_with_os == other.shared_with_os and - self.expected_slot_count == other.expected_slot_count and - self.kind == other.kind + self.type == other.type + and self.path == other.path + and self.shared_with_os == other.shared_with_os + and self.expected_slot_count == other.expected_slot_count + and self.kind == other.kind ) def __hash__(self): - return hash('\0'.join(map(str, (self.type, self.path, self.shared_with_os, self.expected_slot_count, self.kind)))) + return hash( + '\0'.join(map(str, (self.type, self.path, self.shared_with_os, self.expected_slot_count, self.kind))) + ) Domain = collections.namedtuple( - '_Domain', [ + '_Domain', + [ 'domain_name', 'domain_id', 'plan_resolution', @@ -92,38 +97,34 @@ Domain = collections.namedtuple( 'tenants', 'console_initializers', 'bind_slots_to_numa_nodes', - 'config_cookie' - ] + 'config_cookie', + ], ) KiKiMRHost = collections.namedtuple( - '_KiKiMRHost', [ - 'hostname', - 'node_id', - 'drives', - 'ic_port', - 'body', - 'datacenter', - 'rack', - 'host_config_id' - ] + '_KiKiMRHost', ['hostname', 'node_id', 'drives', 'ic_port', 'body', 'datacenter', 'rack', 'host_config_id'] ) DEFAULT_PLAN_RESOLUTION = 10 -DEFAULT_DOMAIN = { - 'domain_name': 'Root', - 'domain_id': 1, - 'plan_resolution': DEFAULT_PLAN_RESOLUTION -} +DEFAULT_DOMAIN = {'domain_name': 'Root', 'domain_id': 1, 'plan_resolution': DEFAULT_PLAN_RESOLUTION} DEFAULT_RESOURCE_BROKER_CPU_LIMIT = 20 -DEFAULT_RESOURCE_BROKER_MEMORY_LIMIT = 16 * (1024 ** 3) +DEFAULT_RESOURCE_BROKER_MEMORY_LIMIT = 16 * (1024**3) class StoragePool(object): def __init__( - self, box_id, erasure, num_groups, filter_properties, - fail_domain_type, kind, name, vdisk_kind, encryption_mode, generation=0 + self, + box_id, + erasure, + num_groups, + filter_properties, + fail_domain_type, + kind, + name, + vdisk_kind, + encryption_mode, + generation=0, ): self._dict_items = {} self.box_id = box_id @@ -151,10 +152,7 @@ class StoragePool(object): class StoragePoolKind(object): - def __init__( - self, box_id, erasure, filter_properties, - fail_domain_type, kind, vdisk_kind, encryption_mode - ): + def __init__(self, box_id, erasure, filter_properties, fail_domain_type, kind, vdisk_kind, encryption_mode): self._dict_items = {} self.box_id = box_id self.erasure = erasure @@ -178,36 +176,32 @@ class StoragePoolKind(object): Subdomain = collections.namedtuple( - '_Subdomain', [ + '_Subdomain', + [ 'domain', 'name', 'plan_resolution', 'mediators', 'coordinators', 'storage_pools', - ] + ], ) DynamicSlot = collections.namedtuple( - '_DynamicSlot', [ + '_DynamicSlot', + [ 'domain', 'id', - ] + ], ) Profile = collections.namedtuple( - '_Profile', [ + '_Profile', + [ 'channels', - ] -) -Channel = collections.namedtuple( - '_Channel', [ - 'erasure', - 'pdisk_type', - 'vdisk_kind', - 'storage_pool_kind' - ] + ], ) +Channel = collections.namedtuple('_Channel', ['erasure', 'pdisk_type', 'vdisk_kind', 'storage_pool_kind']) class StorageUnit(object): @@ -224,19 +218,15 @@ class ComputeUnit(object): class Tenant(object): - def __init__(self, name, storage_units, compute_units=None, overridden_configs=None, shared=False, plan_resolution=None): + def __init__( + self, name, storage_units, compute_units=None, overridden_configs=None, shared=False, plan_resolution=None + ): self.name = name self.overridden_configs = overridden_configs - self.storage_units = tuple( - StorageUnit(**storage_unit_template) - for storage_unit_template in storage_units - ) + self.storage_units = tuple(StorageUnit(**storage_unit_template) for storage_unit_template in storage_units) self.compute_units = tuple() if compute_units: - self.compute_units = tuple( - ComputeUnit(**compute_unit_template) - for compute_unit_template in compute_units - ) + self.compute_units = tuple(ComputeUnit(**compute_unit_template) for compute_unit_template in compute_units) self.shared = shared self.plan_resolution = plan_resolution @@ -274,14 +264,11 @@ def normalize_domain(domain_name): class ClusterDetailsProvider(object): def __init__(self, template, walle_provider, database=None): - self.__cluster_description = _ClusterDescription.from_template( - template) + self.__cluster_description = _ClusterDescription.from_template(template) self.__cluster_description.validate() if database is not None: - self.__cluster_description = self.get_subjective_description( - self.__cluster_description, - database) + self.__cluster_description = self.get_subjective_description(self.__cluster_description, database) self._use_walle = self.__cluster_description.get('use_walle', True) if not walle_provider: @@ -318,17 +305,18 @@ class ClusterDetailsProvider(object): if get_full_tenant_name(domain_of_tenant, current_tenant.get('name')) != tenant: continue - overridden_configurations = current_tenant.get( - 'overridden_configs', {}) - for service in ('nfs', 'nfs_control', 'nbs', 'nbs_control', ): + overridden_configurations = current_tenant.get('overridden_configs', {}) + for service in ( + 'nfs', + 'nfs_control', + 'nbs', + 'nbs_control', + ): if service in overridden_configurations: - overridden_configurations[service]['domain'] = domain_template.get( - 'domain_name') - overridden_configurations[service]['subdomain'] = current_tenant.get( - 'name') + overridden_configurations[service]['domain'] = domain_template.get('domain_name') + overridden_configurations[service]['subdomain'] = current_tenant.get('name') - subjective_description = _ClusterDescription( - dict(objective_description)) + subjective_description = _ClusterDescription(dict(objective_description)) subjective_description.update(overridden_configurations) subjective_description.validate() return subjective_description @@ -400,19 +388,11 @@ class ClusterDetailsProvider(object): return self._hosts futures = [] for node_id, host_description in enumerate(self.__cluster_description.get('hosts'), 1): - futures.append( - self._thread_pool.submit( - self.__collect_host_info, - node_id, - host_description - ) - ) + futures.append(self._thread_pool.submit(self.__collect_host_info, node_id, host_description)) r = [] for f in futures: - r.append( - f.result() - ) + r.append(f.result()) if self._hosts is None: self._hosts = r @@ -420,9 +400,7 @@ class ClusterDetailsProvider(object): @property def system_tablets_node_ids(self): - return [ - host.node_id for host in self.static_bs_group_hosts - ] + return [host.node_id for host in self.static_bs_group_hosts] @property def state_storages(self): @@ -434,8 +412,7 @@ class ClusterDetailsProvider(object): if node_ids: return node_ids - state_storage_node_set = set(self.__cluster_description.get( - 'state_storage', {}).get('node_set', [])) + state_storage_node_set = set(self.__cluster_description.get('state_storage', {}).get('node_set', [])) if len(state_storage_node_set) < 1: return [host.node_id for host in self.static_bs_group_hosts] node_ids = [] @@ -450,8 +427,7 @@ class ClusterDetailsProvider(object): @property def static_bs_group_hosts(self): - static_bs_group_filter = self.__cluster_description.get( - 'static_bs_group_hosts', []) + static_bs_group_filter = self.__cluster_description.get('static_bs_group_hosts', []) matched = [] for host in self.hosts: if host.hostname in static_bs_group_filter or len(static_bs_group_filter) == 0: @@ -491,9 +467,7 @@ class ClusterDetailsProvider(object): HostConfig( host_config_id=host_config['host_config_id'], generation=host_config.get('generation', 0), - drives=tuple( - KiKiMRDrive(**drive) for drive in host_config_drives - ) + drives=tuple(KiKiMRDrive(**drive) for drive in host_config_drives), ) ) return converted_host_configs @@ -512,17 +486,14 @@ class ClusterDetailsProvider(object): for channel in profile.get('channels', []): erasure = channel.get('erasure', str(self.static_erasure)) pdisk_type = channel.get('pdisk_type') - pdisk_type = None if pdisk_type is None else types.PDiskCategory.from_string( - pdisk_type) - channel_storage_pool_kind = channel.get( - 'storage_pool_kind', None) + pdisk_type = None if pdisk_type is None else types.PDiskCategory.from_string(pdisk_type) + channel_storage_pool_kind = channel.get('storage_pool_kind', None) if channel_storage_pool_kind is not None and pdisk_type is None: pdisk_type = None for storage_pool_kind in storage_pool_kinds: if channel_storage_pool_kind == storage_pool_kind.kind: f_properties = storage_pool_kind.filter_properties - pdisk_type = types.PDiskCategory.from_string( - f_properties['type']) + pdisk_type = types.PDiskCategory.from_string(f_properties['type']) assert pdisk_type is not None @@ -534,13 +505,7 @@ class ClusterDetailsProvider(object): storage_pool_kind=channel_storage_pool_kind, ) ) - all_profiles.append( - Profile( - channels=tuple( - profile_channels - ) - ) - ) + all_profiles.append(Profile(channels=tuple(profile_channels))) return tuple(all_profiles) # Domains Stuff @@ -552,8 +517,7 @@ class ClusterDetailsProvider(object): box_id=storage_description.get("box_id", DEFAULT_BOX_ID), erasure=storage_description.get('erasure'), filter_properties=storage_description.get('filter_properties', {}), - fail_domain_type=storage_description.get( - 'fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE), + fail_domain_type=storage_description.get('fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE), vdisk_kind=storage_description.get('vdisk_kind', 'Default'), encryption_mode=storage_description.get('encryption_mode', 0), ) @@ -572,8 +536,7 @@ class ClusterDetailsProvider(object): @property def domains(self): - cl_desc_domains = self.__cluster_description.get( - 'domains', [DEFAULT_DOMAIN]) + cl_desc_domains = self.__cluster_description.get('domains', [DEFAULT_DOMAIN]) domains = [] for domain_id, domain in enumerate(cl_desc_domains, 1): domain_name = domain.get('domain_name') @@ -582,12 +545,12 @@ class ClusterDetailsProvider(object): pool_kind.get('kind'): self.__storage_pool_kind(pool_kind) for pool_kind in domain.get('storage_pool_kinds', []) } - assert len(set(storage_pool_kinds.keys())) == len(storage_pool_kinds.keys()), \ - "required unique kind value in storage_pool_kinds items" + assert len(set(storage_pool_kinds.keys())) == len( + storage_pool_kinds.keys() + ), "required unique kind value in storage_pool_kinds items" storage_pools = [ - self.__storage_pool(storage_pool_kinds, - pool_instance, domain_name) + self.__storage_pool(storage_pool_kinds, pool_instance, domain_name) for pool_instance in domain.get('storage_pools', []) ] @@ -595,14 +558,10 @@ class ClusterDetailsProvider(object): Domain( domain_name=domain_name, domain_id=domain.get('domain_id', domain_id), - mediators=domain.get( - 'mediators', self.__coordinators_count_optimal), - coordinators=domain.get( - 'coordinators', self.__mediators_count_optimal), - allocators=domain.get( - 'allocators', self.__allocators_count_optimal), - plan_resolution=domain.get( - 'plan_resolution', DEFAULT_PLAN_RESOLUTION), + mediators=domain.get('mediators', self.__coordinators_count_optimal), + coordinators=domain.get('coordinators', self.__mediators_count_optimal), + allocators=domain.get('allocators', self.__allocators_count_optimal), + plan_resolution=domain.get('plan_resolution', DEFAULT_PLAN_RESOLUTION), dynamic_slots=domain.get('dynamic_slots', 0), storage_pool_kinds=storage_pool_kinds, storage_pools=storage_pools, @@ -619,8 +578,7 @@ class ClusterDetailsProvider(object): tenants = domain_description.get('databases', []) prepared_tenants = [] for tenant in tenants: - prepared_tenants.append( - Tenant(**tenant)) + prepared_tenants.append(Tenant(**tenant)) return prepared_tenants # Static BlobStorage Group Stuff @@ -636,10 +594,7 @@ class ClusterDetailsProvider(object): @property def fail_domain_type(self): return types.FailDomainType.from_string( - str( - self.__cluster_description.get( - 'fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE) - ) + str(self.__cluster_description.get('fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE)) ) @property @@ -648,8 +603,7 @@ class ClusterDetailsProvider(object): @property def static_erasure(self): - return types.Erasure.from_string( - self.__cluster_description['static_erasure']) + return types.Erasure.from_string(self.__cluster_description['static_erasure']) # Auth txt @property @@ -780,14 +734,12 @@ class ClusterDetailsProvider(object): box_id=DEFAULT_BOX_ID, erasure=storage_pool.get('erasure'), num_groups=storage_pool.get('num_groups'), - filter_properties=storage_pool.get( - 'filter_properties', {}), - fail_domain_type=storage_pool.get( - 'fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE), + filter_properties=storage_pool.get('filter_properties', {}), + fail_domain_type=storage_pool.get('fail_domain_type', DEFAULT_FAIL_DOMAIN_TYPE), kind=storage_pool.get('kind', None), vdisk_kind=storage_pool.get('vdisk_kind', 'Default'), encryption_mode=storage_pool.get('encryption_mode', 0), - generation=storage_pool.get('generation', 0) + generation=storage_pool.get('generation', 0), ) ) @@ -802,14 +754,11 @@ class ClusterDetailsProvider(object): def __storage_pool(self, storage_pool_kinds, storage_pool_instance, domain_name): kind_name = storage_pool_instance.get('kind') - assert kind_name in storage_pool_kinds, \ - "databse storage pools should use pool kinds from storage_pool_kinds section" + assert ( + kind_name in storage_pool_kinds + ), "databse storage pools should use pool kinds from storage_pool_kinds section" - storage_pool_name = "/{domain}:{kind}".format( - domain=domain_name, - kind=kind_name - - ) + storage_pool_name = "/{domain}:{kind}".format(domain=domain_name, kind=kind_name) return StoragePool( name=storage_pool_name, @@ -823,12 +772,7 @@ class ClusterDetailsProvider(object): slots = [] for domain_description in self.__cluster_description.get('domains', []): for slot_id in range(1, int(domain_description.get('dynamic_slots', 0) + 1)): - slots.append( - DynamicSlot( - domain=domain_description.get('domain_name'), - id=slot_id - ) - ) + slots.append(DynamicSlot(domain=domain_description.get('domain_name'), id=slot_id)) return slots @property diff --git a/ydb/tools/cfg/bin/__main__.py b/ydb/tools/cfg/bin/__main__.py index 7d50e8ffdc..a4c6156311 100644 --- a/ydb/tools/cfg/bin/__main__.py +++ b/ydb/tools/cfg/bin/__main__.py @@ -7,35 +7,30 @@ from logging import config as logging_config import yaml -from ydb.tools.cfg.configurator_setup import (get_parser, - parse_optional_arguments) +from ydb.tools.cfg.configurator_setup import get_parser, parse_optional_arguments from ydb.tools.cfg.dynamic import DynamicConfigGenerator from ydb.tools.cfg.static import StaticConfigGenerator from ydb.tools.cfg.utils import write_to_file -logging_config.dictConfig({ - 'version': 1, - 'formatters': { - 'base': { - 'format': '%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s', +logging_config.dictConfig( + { + 'version': 1, + 'formatters': { + 'base': { + 'format': '%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s', + }, }, - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - 'formatter': 'base', - 'stream': sys.stdout, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'level': 'INFO', + 'formatter': 'base', + 'stream': sys.stdout, + }, }, - }, - 'root': { - 'formatter': 'base', - 'level': 'INFO', - 'handlers': ( - 'console', - ) - }, -}) + 'root': {'formatter': 'base', 'level': 'INFO', 'handlers': ('console',)}, + } +) logger = logging.getLogger() @@ -52,19 +47,11 @@ def cfg_generate(args): with open(args.cluster_description, 'r') as yaml_template: cluster_template = yaml.safe_load(yaml_template) - generator = cfg_cls( - cluster_template, - args.binary_path, - args.output_dir, - **kwargs - ) + generator = cfg_cls(cluster_template, args.binary_path, args.output_dir, **kwargs) all_configs = generator.get_all_configs() for cfg_name, cfg_value in all_configs.items(): - write_to_file( - os.path.join(args.output_dir, cfg_name), - cfg_value - ) + write_to_file(os.path.join(args.output_dir, cfg_name), cfg_value) def main(): diff --git a/ydb/tools/cfg/configurator_setup.py b/ydb/tools/cfg/configurator_setup.py index 18041efcc7..981bdd7b5d 100644 --- a/ydb/tools/cfg/configurator_setup.py +++ b/ydb/tools/cfg/configurator_setup.py @@ -39,66 +39,33 @@ def get_parser(generate_func, extra_cfg_arguments=[]): ) subparsers = parser.add_subparsers(help='sub-command help') - parser_cfg = subparsers.add_parser( - 'cfg', help='run static config generation') + parser_cfg = subparsers.add_parser('cfg', help='run static config generation') parser_cfg.add_argument( - "cluster_description", type=str, + "cluster_description", + type=str, help="Path to the cluster description", ) + parser_cfg.add_argument("binary_path", type=str, help="Path to the KiKiMR's binary") + parser_cfg.add_argument("output_dir", type=str, help="Path to store configuration files") + parser_cfg.add_argument("--seed", type=int, help='Force cfg command to use special seed in instantiate random') + parser_cfg.add_argument("--grpc-endpoint", type=str, help='gRPC endpoint to use in scripts') parser_cfg.add_argument( - "binary_path", type=str, - help="Path to the KiKiMR's binary" + '--database', type=str, help='Database to serve (possible options: name of domain or no for pure storage nodes)' ) parser_cfg.add_argument( - "output_dir", type=str, - help="Path to store configuration files" + '--tenant', type=str, help='Tenant to serve (possible options: name of domain or no for pure storage nodes)' ) + parser_cfg.add_argument('--enable-cores', action='store_true', help='Enables coredumps') parser_cfg.add_argument( - "--seed", type=int, - help='Force cfg command to use special seed in instantiate random' - ) - parser_cfg.add_argument( - "--grpc-endpoint", type=str, - help='gRPC endpoint to use in scripts' - ) - parser_cfg.add_argument( - '--database', type=str, - help='Database to serve (possible options: name of domain or no for pure storage nodes)' - ) - parser_cfg.add_argument( - '--tenant', type=str, - help='Tenant to serve (possible options: name of domain or no for pure storage nodes)' - ) - parser_cfg.add_argument( - '--enable-cores', action='store_true', - help='Enables coredumps' - ) - parser_cfg.add_argument( - '--dynamic-node', action='store_true', - help='Indicates that configuration should be generated for dynamic node' - ) - parser_cfg.add_argument( - '--node-broker-port', type=str, - help='Node Broker Port to use' - ) - parser_cfg.add_argument( - '--cfg-home', type=str, - help='Configuration home directory' - ) - parser_cfg.add_argument( - '--binaries-home', type=str, - help='Binaries home directory' - ) - parser_cfg.add_argument( - '--local-binary-path', type=str, - help='Path to kikimr binary current host' + '--dynamic-node', action='store_true', help='Indicates that configuration should be generated for dynamic node' ) + parser_cfg.add_argument('--node-broker-port', type=str, help='Node Broker Port to use') + parser_cfg.add_argument('--cfg-home', type=str, help='Configuration home directory') + parser_cfg.add_argument('--binaries-home', type=str, help='Binaries home directory') + parser_cfg.add_argument('--local-binary-path', type=str, help='Path to kikimr binary current host') for port in ('grpc-port', 'ic-port', 'mon-port', 'mbus-port', 'sqs-port'): - parser_cfg.add_argument( - '--%s' % port, type=str, - help='Port to be used during KiKiMR server start' - ) + parser_cfg.add_argument('--%s' % port, type=str, help='Port to be used during KiKiMR server start') for v in extra_cfg_arguments: parser_cfg.add_argument( @@ -109,28 +76,18 @@ def get_parser(generate_func, extra_cfg_arguments=[]): argument_group = parser_cfg.add_mutually_exclusive_group() argument_group.add_argument( - '--static', action='store_true', - help='Forces cfg command to generate static configuration' - ) - argument_group.add_argument( - '--dynamic', action='store_true', - help='Forces cfg command to generate dynamic configuration' - ) - argument_group.add_argument( - '--nbs', action='store_true', - help='Forces cfg command to generate NBS configuration' + '--static', action='store_true', help='Forces cfg command to generate static configuration' ) argument_group.add_argument( - '--nbs-control', action='store_true', - help='Forces cfg command to generate NBS Control configuration' + '--dynamic', action='store_true', help='Forces cfg command to generate dynamic configuration' ) + argument_group.add_argument('--nbs', action='store_true', help='Forces cfg command to generate NBS configuration') argument_group.add_argument( - '--nfs', action='store_true', - help='Forces cfg command to generate NFS configuration' + '--nbs-control', action='store_true', help='Forces cfg command to generate NBS Control configuration' ) + argument_group.add_argument('--nfs', action='store_true', help='Forces cfg command to generate NFS configuration') argument_group.add_argument( - '--nfs-control', action='store_true', - help='Forces cfg command to generate NFS Control configuration' + '--nfs-control', action='store_true', help='Forces cfg command to generate NFS Control configuration' ) parser_cfg.set_defaults(func=generate_func) diff --git a/ydb/tools/cfg/dynamic.py b/ydb/tools/cfg/dynamic.py index 8bd8aebc85..f786b654a0 100644 --- a/ydb/tools/cfg/dynamic.py +++ b/ydb/tools/cfg/dynamic.py @@ -4,21 +4,24 @@ import itertools import os from contrib.ydb.core.protos import blobstorage_config_pb2 as bs_config -from contrib.ydb.core.protos import \ - blobstorage_pdisk_config_pb2 as pdisk_config +from contrib.ydb.core.protos import blobstorage_pdisk_config_pb2 as pdisk_config from contrib.ydb.core.protos import flat_scheme_op_pb2 as flat_scheme_op from contrib.ydb.core.protos import msgbus_pb2 as msgbus from contrib.ydb.core.protos import tx_proxy_pb2 as tx_proxy from ydb.tools.cfg import base, static, utils -from ydb.tools.cfg.types import (DistinctionLevels, Erasure, FailDomainType, - PDiskCategory) +from ydb.tools.cfg.types import DistinctionLevels, Erasure, FailDomainType, PDiskCategory class DynamicConfigGenerator(object): def __init__( - self, template, binary_path, output_dir, grpc_endpoint=None, - local_binary_path=None, walle_provider=None, - **kwargs + self, + template, + binary_path, + output_dir, + grpc_endpoint=None, + local_binary_path=None, + walle_provider=None, + **kwargs ): self._template = template self._binary_path = binary_path @@ -29,7 +32,9 @@ class DynamicConfigGenerator(object): self._grpc_endpoint = grpc_endpoint self.__define_storage_pools_request = None self.__configure_request = None - self.__static_config = static.StaticConfigGenerator(template, binary_path, output_dir, walle_provider=walle_provider, local_binary_path=local_binary_path) + self.__static_config = static.StaticConfigGenerator( + template, binary_path, output_dir, walle_provider=walle_provider, local_binary_path=local_binary_path + ) @property def grpc_endpoint(self): @@ -49,16 +54,23 @@ class DynamicConfigGenerator(object): "config", "invoke", "--proto-file", - os.path.join( - self._output_dir, - proto_file - ) + os.path.join(self._output_dir, proto_file), ] ) def __add_storage_pool( - self, box_id, storage_pool_id, erasure, filter_properties, num_groups, fail_domain_type, kind=None, - name=None, vdisk_kind='Default', encryption_mode=0, generation=0, + self, + box_id, + storage_pool_id, + erasure, + filter_properties, + num_groups, + fail_domain_type, + kind=None, + name=None, + vdisk_kind='Default', + encryption_mode=0, + generation=0, ): cmd = self.__define_storage_pools_request.Command.add() cmd.DefineStoragePool.BoxId = box_id @@ -104,10 +116,7 @@ class DynamicConfigGenerator(object): "db", "schema", "execute", - os.path.join( - self._output_dir, - proto_file - ), + os.path.join(self._output_dir, proto_file), ] ) @@ -122,10 +131,7 @@ class DynamicConfigGenerator(object): "execute", "--domain=%s" % domain_name, "--retry=10", - os.path.join( - self._output_dir, - proto_file - ), + os.path.join(self._output_dir, proto_file), ] ) @@ -140,34 +146,29 @@ class DynamicConfigGenerator(object): for init_id, filename in enumerate(domain.console_initializers, 1): commands.append(self.__cms_init_cmd('Configure-%s-init-%d.txt' % (name, init_id), name)) - return '\n'.join( - commands - ) + return '\n'.join(commands) def init_storage_commands(self): return '\n'.join( [ "set -eu", self.__init_storage_command("DefineBox.txt"), - self.__init_storage_command("DefineStoragePools.txt") + self.__init_storage_command("DefineStoragePools.txt"), ] ) def init_compute_commands(self): commands = ["set -eu"] - return '\n'.join( - commands - ) + return '\n'.join(commands) def init_root_storage(self): commands = ["set -eu"] commands += [ self.__bind_storage_with_root('BindRootStorageRequest-%s.txt' % domain.domain_name) - for domain in self._cluster_details.domains if len(domain.storage_pools) + for domain in self._cluster_details.domains + if len(domain.storage_pools) ] - return '\n'.join( - commands - ) + return '\n'.join(commands) @property def cms_init_cmd(self): @@ -207,7 +208,8 @@ class DynamicConfigGenerator(object): if at_least_one_host_config_defined: raise RuntimeError( - "At least one host config defined manually, but you still use drives directly attached to hosts") + "At least one host config defined manually, but you still use drives directly attached to hosts" + ) host_config_id = next(host_config_id_iter) cmd = define_box_request.Command.add() @@ -236,19 +238,13 @@ class DynamicConfigGenerator(object): if self._cluster_details.storage_pools_deprecated: for storage_pool in self._cluster_details.storage_pools_deprecated: - self.__add_storage_pool( - storage_pool_id=next(storage_pool_id), - **storage_pool.to_dict() - ) + self.__add_storage_pool(storage_pool_id=next(storage_pool_id), **storage_pool.to_dict()) # for tablets in domain lets make pools # but it is not supposed to make tablets in domain directly for domain in self._cluster_details.domains: for storage_pool in domain.storage_pools: - self.__add_storage_pool( - storage_pool_id=next(storage_pool_id), - **storage_pool.to_dict() - ) + self.__add_storage_pool(storage_pool_id=next(storage_pool_id), **storage_pool.to_dict()) return self.__define_storage_pools_request @@ -283,7 +279,9 @@ class DynamicConfigGenerator(object): if domain.config_cookie: action.AddConfigItem.ConfigItem.Cookie = domain.config_cookie - configure_request.ConfigureRequest.Actions.add().RemoveConfigItems.CookieFilter.Cookies.append(domain.config_cookie) + configure_request.ConfigureRequest.Actions.add().RemoveConfigItems.CookieFilter.Cookies.append( + domain.config_cookie + ) action = configure_request.ConfigureRequest.Actions.add() action.AddConfigItem.ConfigItem.UsageScope.TenantAndNodeTypeFilter.Tenant = "dynamic" @@ -306,8 +304,7 @@ class DynamicConfigGenerator(object): @staticmethod def _construct_create_tenant_request(domain, tenant): console_request = msgbus.TConsoleRequest() - console_request.CreateTenantRequest.Request.path = os.path.join( - '/', domain.domain_name, tenant.name) + console_request.CreateTenantRequest.Request.path = os.path.join('/', domain.domain_name, tenant.name) if tenant.shared: resources = console_request.CreateTenantRequest.Request.shared_resources @@ -337,8 +334,7 @@ class DynamicConfigGenerator(object): for domain in self._cluster_details.domains: for tenant in domain.tenants: tenants_count += 1 - files['CreateTenant-%d.txt' % next(tn_id)] = self._construct_create_tenant_request( - domain, tenant) + files['CreateTenant-%d.txt' % next(tn_id)] = self._construct_create_tenant_request(domain, tenant) return tenants_count, files def get_create_tenant_commands(self): @@ -346,10 +342,7 @@ class DynamicConfigGenerator(object): commands = [] for domain in self._cluster_details.domains: for _ in domain.tenants: - commands.append( - self.__cms_init_cmd( - 'CreateTenant-%d.txt' % next(tn_id), - domain.domain_name)) + commands.append(self.__cms_init_cmd('CreateTenant-%d.txt' % next(tn_id), domain.domain_name)) commands.append('exit 0') return "\n".join(commands) @@ -358,9 +351,10 @@ class DynamicConfigGenerator(object): all_configs = self.get_storage_requests() all_configs.update( { - 'BindRootStorageRequest-%s.txt' % domain.domain_name: utils.message_to_string( - self.make_bind_root_storage_request(domain)) - for domain in self._cluster_details.domains if len(domain.storage_pools) + 'BindRootStorageRequest-%s.txt' + % domain.domain_name: utils.message_to_string(self.make_bind_root_storage_request(domain)) + for domain in self._cluster_details.domains + if len(domain.storage_pools) } ) all_configs.update( diff --git a/ydb/tools/cfg/static.py b/ydb/tools/cfg/static.py index b07f3c9bff..f9da81776a 100644 --- a/ydb/tools/cfg/static.py +++ b/ydb/tools/cfg/static.py @@ -8,32 +8,50 @@ import subprocess import tempfile import yaml -from contrib.ydb.core.fq.libs.config.protos.fq_config_pb2 import \ - TConfig as TFederatedQueryConfig +from contrib.ydb.core.fq.libs.config.protos.fq_config_pb2 import TConfig as TFederatedQueryConfig from google.protobuf import json_format -from contrib.ydb.core.protos import (auth_pb2, blobstorage_vdisk_config_pb2, - cms_pb2, config_pb2, feature_flags_pb2, - netclassifier_pb2, pqconfig_pb2, - resource_broker_pb2) +from contrib.ydb.core.protos import ( + auth_pb2, + blobstorage_vdisk_config_pb2, + cms_pb2, + config_pb2, + feature_flags_pb2, + netclassifier_pb2, + pqconfig_pb2, + resource_broker_pb2, +) from ydb.tools.cfg import base, types, utils -from ydb.tools.cfg.templates import (dynamic_cfg_new_style, - kikimr_cfg_for_dynamic_node, - kikimr_cfg_for_dynamic_slot, - kikimr_cfg_for_static_node, - kikimr_cfg_for_static_node_new_style) +from ydb.tools.cfg.templates import ( + dynamic_cfg_new_style, + kikimr_cfg_for_dynamic_node, + kikimr_cfg_for_dynamic_slot, + kikimr_cfg_for_static_node, + kikimr_cfg_for_static_node_new_style, +) logger = logging.getLogger(__name__) class StaticConfigGenerator(object): def __init__( - self, template, binary_path, output_dir, database=None, - node_broker_port=2135, ic_port=19001, walle_provider=None, - grpc_port=2135, mon_port=8765, cfg_home='/Berkanavt/kikimr', - sqs_port=8771, enable_cores=False, enable_cms_config_cache=False, - local_binary_path=None, skip_location=False, - **kwargs + self, + template, + binary_path, + output_dir, + database=None, + node_broker_port=2135, + ic_port=19001, + walle_provider=None, + grpc_port=2135, + mon_port=8765, + cfg_home='/Berkanavt/kikimr', + sqs_port=8771, + enable_cores=False, + enable_cms_config_cache=False, + local_binary_path=None, + skip_location=False, + **kwargs ): self.__proto_configs = {} self.__binary_path = binary_path @@ -81,16 +99,23 @@ class StaticConfigGenerator(object): 'pqcd.txt': None, 'failure_injection.txt': None, } - self.__optional_config_files = set(( - 'rb.txt', - 'metering.txt', - 'audit.txt', - 'fq.txt', - 'failure_injection.txt', - )) + self.__optional_config_files = set( + ( + 'rb.txt', + 'metering.txt', + 'audit.txt', + 'fq.txt', + 'failure_injection.txt', + ) + ) self._enable_cms_config_cache = template.get('enable_cms_config_cache', enable_cms_config_cache) if 'tracing' in template: - self.__tracing = (template['tracing']['host'], template['tracing']['port'], template['tracing']['root_ca'], template['tracing']['service_name']) + self.__tracing = ( + template['tracing']['host'], + template['tracing']['port'], + template['tracing']['root_ca'], + template['tracing']['service_name'], + ) else: self.__tracing = None self.__write_mbus_settings_to_kikimr_cfg = False @@ -129,11 +154,17 @@ class StaticConfigGenerator(object): @property def feature_flags_txt(self): - return self.__proto_config('feature_flags.txt', feature_flags_pb2.TFeatureFlags, self.__cluster_details.get_service("features")) + return self.__proto_config( + 'feature_flags.txt', feature_flags_pb2.TFeatureFlags, self.__cluster_details.get_service("features") + ) @property def failure_injection_txt(self): - return self.__proto_config('failure_injection.txt', config_pb2.TFailureInjectionConfig, self.__cluster_details.get_service("failure_injection_config")) + return self.__proto_config( + 'failure_injection.txt', + config_pb2.TFailureInjectionConfig, + self.__cluster_details.get_service("failure_injection_config"), + ) @property def failure_injection_txt_enabled(self): @@ -145,11 +176,17 @@ class StaticConfigGenerator(object): @property def netclassifier_txt(self): - return self.__proto_config("netclassifier.txt", netclassifier_pb2.TNetClassifierConfig, self.__cluster_details.get_service("netclassifier")) + return self.__proto_config( + "netclassifier.txt", + netclassifier_pb2.TNetClassifierConfig, + self.__cluster_details.get_service("netclassifier"), + ) @property def pqcd_txt(self): - return self.__proto_config("pqcd.txt", pqconfig_pb2.TPQClusterDiscoveryConfig, self.__cluster_details.get_service("pqclusterdiscovery")) + return self.__proto_config( + "pqcd.txt", pqconfig_pb2.TPQClusterDiscoveryConfig, self.__cluster_details.get_service("pqclusterdiscovery") + ) @property def ic_txt(self): @@ -161,7 +198,9 @@ class StaticConfigGenerator(object): @property def dyn_ns_txt(self): - return self.__proto_config('dyn_ns.txt', config_pb2.TDynamicNameserviceConfig, self.__cluster_details.dynamicnameservice_config) + return self.__proto_config( + 'dyn_ns.txt', config_pb2.TDynamicNameserviceConfig, self.__cluster_details.dynamicnameservice_config + ) @property def log_txt(self): @@ -173,7 +212,9 @@ class StaticConfigGenerator(object): @property def vdisks_txt(self): - return self.__proto_config('vdisks.txt', blobstorage_vdisk_config_pb2.TAllVDiskKinds, self.__cluster_details.vdisk_config) + return self.__proto_config( + 'vdisks.txt', blobstorage_vdisk_config_pb2.TAllVDiskKinds, self.__cluster_details.vdisk_config + ) @property def sqs_txt(self): @@ -185,7 +226,9 @@ class StaticConfigGenerator(object): @property def rb_txt(self): - return self.__proto_config('rb.txt', resource_broker_pb2.TResourceBrokerConfig, self.__cluster_details.get_service('resource_broker')) + return self.__proto_config( + 'rb.txt', resource_broker_pb2.TResourceBrokerConfig, self.__cluster_details.get_service('resource_broker') + ) @property def rb_txt_enabled(self): @@ -193,7 +236,9 @@ class StaticConfigGenerator(object): @property def metering_txt(self): - return self.__proto_config('metering.txt', config_pb2.TMeteringConfig, self.__cluster_details.get_service("metering")) + return self.__proto_config( + 'metering.txt', config_pb2.TMeteringConfig, self.__cluster_details.get_service("metering") + ) @property def metering_txt_enabled(self): @@ -232,9 +277,14 @@ class StaticConfigGenerator(object): def kikimr_cfg(self): if self.__is_dynamic_node: return kikimr_cfg_for_dynamic_node( - self.__node_broker_port, self._database, - self.__ic_port, self.__mon_port, - self.__kikimr_home, self.__sqs_port, self.sqs_txt.EnableSqs, self._enable_cores, + self.__node_broker_port, + self._database, + self.__ic_port, + self.__mon_port, + self.__kikimr_home, + self.__sqs_port, + self.sqs_txt.EnableSqs, + self._enable_cores, self.__cluster_details.default_log_level, mon_address=self.__cluster_details.monitor_address, cert_params=self.__cluster_details.ic_cert_params, @@ -242,7 +292,7 @@ class StaticConfigGenerator(object): rb_txt_enabled=self.rb_txt_enabled, metering_txt_enabled=self.metering_txt_enabled, audit_txt_enabled=self.audit_txt_enabled, - fq_txt_enabled=self.fq_txt_enabled + fq_txt_enabled=self.fq_txt_enabled, ) if self.__cluster_details.use_new_style_kikimr_cfg: @@ -258,8 +308,11 @@ class StaticConfigGenerator(object): ) return kikimr_cfg_for_static_node( - self._database, self.__ic_port, - self.__mon_port, self.__kikimr_home, self.pq_txt.Enabled, + self._database, + self.__ic_port, + self.__mon_port, + self.__kikimr_home, + self.pq_txt.Enabled, self._enable_cores, self.__cluster_details.default_log_level, mon_address=self.__cluster_details.monitor_address, @@ -368,10 +421,7 @@ class StaticConfigGenerator(object): system_tablets_info[tablet_type] = [] system_tablets_info[tablet_type].append( - { - "info": {"tablet_id": tablet["info"]["tablet_id"]}, - "node": tablet["node"] - } + {"info": {"tablet_id": tablet["info"]["tablet_id"]}, "node": tablet["node"]} ) del normalized_config["bootstrap_config"]["tablet"] @@ -510,13 +560,9 @@ class StaticConfigGenerator(object): for channel_id in range(int(number_of_channels)): channel = tablet.Info.Channels.add( - Channel=channel_id, - ChannelErasureName=str(self.__cluster_details.static_erasure) - ) - channel.History.add( - FromGeneration=0, - GroupID=0 + Channel=channel_id, ChannelErasureName=str(self.__cluster_details.static_erasure) ) + channel.History.add(FromGeneration=0, GroupID=0) @property def __tablet_types(self): @@ -548,11 +594,7 @@ class StaticConfigGenerator(object): for tablet_type, tablet_count in self.__system_tablets: for index in range(int(tablet_count)): - self.__add_tablet( - tablet_type, - index, - self.__cluster_details.system_tablets_node_ids - ) + self.__add_tablet(tablet_type, index, self.__cluster_details.system_tablets_node_ids) if self.__cluster_details.shared_cache_memory_limit is not None: boot_txt = self.__proto_configs["boot.txt"] @@ -683,13 +725,17 @@ class StaticConfigGenerator(object): if self.__cluster_details.nw_cache_file_path is not None: self.__proto_configs['bs.txt'].CacheFilePath = self.__cluster_details.nw_cache_file_path - def _read_generated_bs_config(self, static_erasure, min_fail_domains, static_pdisk_type, fail_domain_type, bs_format_config): + def _read_generated_bs_config( + self, static_erasure, min_fail_domains, static_pdisk_type, fail_domain_type, bs_format_config + ): result = config_pb2.TBlobStorageConfig() with tempfile.NamedTemporaryFile(delete=True) as t_file: utils.write_proto_to_file(t_file.name, bs_format_config) - rx_begin, rx_end, dx_begin, dx_end = types.DistinctionLevels[types.FailDomainType.from_string(fail_domain_type)] + rx_begin, rx_end, dx_begin, dx_end = types.DistinctionLevels[ + types.FailDomainType.from_string(fail_domain_type) + ] cmd_base = [ self.__local_binary_path, @@ -697,25 +743,42 @@ class StaticConfigGenerator(object): "bs", "genconfig", "static", - "--bs-format-file", t_file.name, - "--erasure", static_erasure, - "--avdomain", "1", - "--faildomains", min_fail_domains, - "--vdisks", "1", - "--pdisktype", static_pdisk_type, + "--bs-format-file", + t_file.name, + "--erasure", + static_erasure, + "--avdomain", + "1", + "--faildomains", + min_fail_domains, + "--vdisks", + "1", + "--pdisktype", + static_pdisk_type, ] try: - output = subprocess.check_output(cmd_base + [ - "--ring-level-begin", str(rx_begin), - "--ring-level-end", str(rx_end), - "--domain-level-begin", str(dx_begin), - "--domain-level-end", str(dx_end), - ]) + output = subprocess.check_output( + cmd_base + + [ + "--ring-level-begin", + str(rx_begin), + "--ring-level-end", + str(rx_end), + "--domain-level-begin", + str(dx_begin), + "--domain-level-end", + str(dx_end), + ] + ) except subprocess.CalledProcessError: - output = subprocess.check_output(cmd_base + [ - "--dx", fail_domain_type, - ]) + output = subprocess.check_output( + cmd_base + + [ + "--dx", + fail_domain_type, + ] + ) utils.read_message_from_string(output, result) @@ -745,15 +808,13 @@ class StaticConfigGenerator(object): profile.Channel.add( ErasureSpecies=str(self.__cluster_details.static_erasure), PDiskCategory=chosen_category, - VDiskCategory=str(types.VDiskCategory.Default) + VDiskCategory=str(types.VDiskCategory.Default), ) for user_profile in self.__cluster_details.tablet_profiles: - profile = channels_config.Profile.add() profile.ProfileId = next(profile_id) for user_profile_channel in user_profile.channels: - params = { 'ErasureSpecies': str(user_profile_channel.erasure), 'PDiskCategory': user_profile_channel.pdisk_type, @@ -775,9 +836,8 @@ class StaticConfigGenerator(object): n_to_select_candidate = nodes_count else: raise RuntimeError( - "Unable to configure state storage, n to select %d > length of hosts %d" % ( - n_to_select_candidate, nodes_count - ) + "Unable to configure state storage, n to select %d > length of hosts %d" + % (n_to_select_candidate, nodes_count) ) return n_to_select_candidate @@ -809,9 +869,7 @@ class StaticConfigGenerator(object): domain_id = domain_description.domain_id domain_name = domain_description.domain_name domain = domains_config.Domain.add( - Name=domain_name, - DomainId=domain_id, - PlanResolution=domain_description.plan_resolution + Name=domain_name, DomainId=domain_id, PlanResolution=domain_description.plan_resolution ) domain.SSId.append(domain_id) domain.HiveUid.append(domain_id) @@ -824,24 +882,18 @@ class StaticConfigGenerator(object): domain.SchemeRoot = schemeroot domain.ExplicitCoordinators.extend( - [tablet_types.FLAT_TX_COORDINATOR.tablet_id_for(i) - for i in range(int(domain_description.coordinators))] + [tablet_types.FLAT_TX_COORDINATOR.tablet_id_for(i) for i in range(int(domain_description.coordinators))] ) domain.ExplicitMediators.extend( - [tablet_types.TX_MEDIATOR.tablet_id_for(i) - for i in range(int(domain_description.mediators))] + [tablet_types.TX_MEDIATOR.tablet_id_for(i) for i in range(int(domain_description.mediators))] ) domain.ExplicitAllocators.extend( - [tablet_types.TX_ALLOCATOR.tablet_id_for(i) - for i in range(int(domain_description.allocators))] + [tablet_types.TX_ALLOCATOR.tablet_id_for(i) for i in range(int(domain_description.allocators))] ) self._configure_statestorages(domains_config, domain_id) - domains_config.HiveConfig.add( - HiveUid=domain_id, - Hive=tablet_types.FLAT_HIVE.tablet_id_for(0) - ) + domains_config.HiveConfig.add(HiveUid=domain_id, Hive=tablet_types.FLAT_HIVE.tablet_id_for(0)) for pool_kind in domain_description.storage_pool_kinds.values(): pool_type = domain.StoragePoolTypes.add(Kind=pool_kind.kind) @@ -868,14 +920,10 @@ class StaticConfigGenerator(object): if 'type' in pool_kind.filter_properties: pdisk_type = pool_kind.filter_properties['type'] pdisk_category = int(types.PDiskCategory.from_string(pdisk_type)) - pdisk_filter.Property.add( - Type=pdisk_category - ) + pdisk_filter.Property.add(Type=pdisk_category) if "SharedWithOs" in pool_kind.filter_properties: - pdisk_filter.Property.add( - SharedWithOs=pool_kind.filter_properties["SharedWithOs"] - ) + pdisk_filter.Property.add(SharedWithOs=pool_kind.filter_properties["SharedWithOs"]) def _get_base_statestorage(self, domains_cfg, ss): ssid = ss.get('ssid', None) @@ -934,8 +982,7 @@ class StaticConfigGenerator(object): rack_sizes = collections.Counter() for node in self.__cluster_details.hosts: rack_sizes[node.rack] += 1 - racks[node.rack].append( - node) + racks[node.rack].append(node) chosen_racks = [] dc_limit = n_to_select == 9 @@ -985,9 +1032,7 @@ class StaticConfigGenerator(object): return if self.__cluster_details.allow_incorrect_state_storage: - logger.warning( - "Using unsafe option: " - "state storage in the cluster is probably broken") + logger.warning("Using unsafe option: " "state storage in the cluster is probably broken") state_storage_cfg.Ring.NToSelect = self.__n_to_select state_storage_cfg.Ring.Node.extend(self.__cluster_details.state_storage_node_ids) return @@ -1050,7 +1095,6 @@ class StaticConfigGenerator(object): node.Location.Body = int(host.body) if self.__cluster_details.use_cluster_uuid: - accepted_uuids = self.__cluster_details.accepted_cluster_uuids cluster_uuid = self.__cluster_details.cluster_uuid cluster_uuid = 'ydb:{}'.format(utils.uuid()) if cluster_uuid is None else cluster_uuid @@ -1063,7 +1107,8 @@ class StaticConfigGenerator(object): if self.__cluster_details.sys_preset_name is not None: utils.read_from_resource( self.__proto_configs['sys.txt'], - 'sys', self.__cluster_details.sys_preset_name, + 'sys', + self.__cluster_details.sys_preset_name, ) elif self.__cluster_details.use_auto_config: sys_config = self.__proto_configs['sys.txt'] @@ -1083,7 +1128,12 @@ class StaticConfigGenerator(object): def __generate_tracing_txt(self): pb = config_pb2.TAppConfig() if self.__tracing: - pb.TracingConfig.Host, pb.TracingConfig.Port, pb.TracingConfig.RootCA, pb.TracingConfig.ServiceName = self.__tracing + ( + pb.TracingConfig.Host, + pb.TracingConfig.Port, + pb.TracingConfig.RootCA, + pb.TracingConfig.ServiceName, + ) = self.__tracing self.__proto_configs['tracing.txt'] = pb def __generate_sys_txt_advanced(self): @@ -1102,7 +1152,7 @@ class StaticConfigGenerator(object): 'Threads': 16, 'SpinThreshold': 1, 'Name': "User", - 'ExecutorUser': 'UserExecutor' + 'ExecutorUser': 'UserExecutor', }, { 'Type': sys_config.TExecutor.EType.Value('BASIC'), @@ -1123,7 +1173,7 @@ class StaticConfigGenerator(object): 'SpinThreshold': 10, 'Name': "IC", 'TimePerMailboxMicroSecs': 100, - 'ExecutorUser': 'Interconnect' + 'ExecutorUser': 'Interconnect', }, ] scheduler = { @@ -1176,5 +1226,6 @@ class StaticConfigGenerator(object): def dynamic_server_common_args(self): if self.__cluster_details.use_new_style_kikimr_cfg: return dynamic_cfg_new_style(self._enable_cores) - return kikimr_cfg_for_dynamic_slot(self._enable_cores, self._enable_cms_config_cache, - cert_params=self.__cluster_details.ic_cert_params) + return kikimr_cfg_for_dynamic_slot( + self._enable_cores, self._enable_cms_config_cache, cert_params=self.__cluster_details.ic_cert_params + ) diff --git a/ydb/tools/cfg/templates.py b/ydb/tools/cfg/templates.py index f88ee6a15c..7f509afe5e 100644 --- a/ydb/tools/cfg/templates.py +++ b/ydb/tools/cfg/templates.py @@ -173,21 +173,28 @@ DEFAULT_KIKIMR_MBUS_MAX_MESSAGE_SIZE = 140000000 def local_vars( - tenant, node_broker_port=None, - ic_port=19001, mon_port=8765, grpc_port=2135, - mbus_port=2134, - kikimr_home='/Berkanavt/kikimr', - kikimr_binaries_base_path='/Berkanavt/kikimr', - pq_enable=False, sqs_port=8771, sqs_enable=False, - enable_cores=False, default_log_level=3, mon_address="", - cert_params=None, - rb_txt_enabled=False, - metering_txt_enabled=False, - audit_txt_enabled=False, - yql_txt_enabled=False, - fq_txt_enabled=False, - new_style_kikimr_cfg=False, - mbus_enabled=False, + tenant, + node_broker_port=None, + ic_port=19001, + mon_port=8765, + grpc_port=2135, + mbus_port=2134, + kikimr_home='/Berkanavt/kikimr', + kikimr_binaries_base_path='/Berkanavt/kikimr', + pq_enable=False, + sqs_port=8771, + sqs_enable=False, + enable_cores=False, + default_log_level=3, + mon_address="", + cert_params=None, + rb_txt_enabled=False, + metering_txt_enabled=False, + audit_txt_enabled=False, + yql_txt_enabled=False, + fq_txt_enabled=False, + new_style_kikimr_cfg=False, + mbus_enabled=False, ): cur_vars = [] if enable_cores: @@ -206,9 +213,7 @@ def local_vars( if node_broker_port: cur_vars.append(('kikimr_node_broker_port', node_broker_port)) - cur_vars.append( - ('kikimr_mon_address', mon_address) - ) + cur_vars.append(('kikimr_mon_address', mon_address)) if default_log_level: cur_vars.extend( @@ -264,11 +269,7 @@ def local_vars( cur_vars.append(("kikimr_mbus_infly_bytes", DEFAULT_KIKIMR_MBUS_INFLY_BYTES)) cur_vars.append(("kikimr_mbus_max_message_size", DEFAULT_KIKIMR_MBUS_MAX_MESSAGE_SIZE)) - return '\n'.join( - [ - '%s="%s"' % (cur_var[0], cur_var[1]) for cur_var in cur_vars - ] - ) + return '\n'.join(['%s="%s"' % (cur_var[0], cur_var[1]) for cur_var in cur_vars]) def tenant_argument(tenant=None): @@ -280,55 +281,87 @@ def pq_enable_argument(pq_enable=False): def sqs_arguments(sqs_enable=False): - return '' if not sqs_enable else '\n'.join( - [ - 'kikimr_arg="${kikimr_arg}${kikimr_sqs_file:+ --sqs-file ${kikimr_sqs_file}}"', - 'kikimr_arg="${kikimr_arg}${kikimr_sqs_port:+ --sqs-port ${kikimr_sqs_port}}"' - ] + return ( + '' + if not sqs_enable + else '\n'.join( + [ + 'kikimr_arg="${kikimr_arg}${kikimr_sqs_file:+ --sqs-file ${kikimr_sqs_file}}"', + 'kikimr_arg="${kikimr_arg}${kikimr_sqs_port:+ --sqs-port ${kikimr_sqs_port}}"', + ] + ) ) def rb_arguments(rb_txt_enabled=False): - return [] if not rb_txt_enabled else [ - 'kikimr_arg="${kikimr_arg}${kikimr_rb_file:+ --rb-file ${kikimr_rb_file}}"', - ] + return ( + [] + if not rb_txt_enabled + else [ + 'kikimr_arg="${kikimr_arg}${kikimr_rb_file:+ --rb-file ${kikimr_rb_file}}"', + ] + ) def metering_arguments(metering_txt_enabled=False): - return [] if not metering_txt_enabled else [ - 'kikimr_arg="${kikimr_arg}${kikimr_metering_file:+ --metering-file ${kikimr_metering_file}}"', - ] + return ( + [] + if not metering_txt_enabled + else [ + 'kikimr_arg="${kikimr_arg}${kikimr_metering_file:+ --metering-file ${kikimr_metering_file}}"', + ] + ) def audit_arguments(audit_txt_enabled=False): - return [] if not audit_txt_enabled else [ - 'kikimr_arg="${kikimr_arg}${kikimr_audit_file:+ --audit-file ${kikimr_audit_file}}"', - ] + return ( + [] + if not audit_txt_enabled + else [ + 'kikimr_arg="${kikimr_arg}${kikimr_audit_file:+ --audit-file ${kikimr_audit_file}}"', + ] + ) def yql_arguments(yql_txt_enabled=False): - return [] if not yql_txt_enabled else [ - 'kikimr_arg="${kikimr_arg}${kikimr_yql_file:+ --yql-file ${kikimr_yql_file}}"', - ] + return ( + [] + if not yql_txt_enabled + else [ + 'kikimr_arg="${kikimr_arg}${kikimr_yql_file:+ --yql-file ${kikimr_yql_file}}"', + ] + ) def cms_config_cache_argument_for_dynamic_nodes(enable_cms_config_cache=False): - return '' if not enable_cms_config_cache else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}_${kikimr_ic_port}/cache/cached_cfg.txt"' + return ( + '' + if not enable_cms_config_cache + else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}_${kikimr_ic_port}/cache/cached_cfg.txt"' + ) def cms_config_cache_argument_for_static_nodes(enable_cms_config_cache=False): - return '' if not enable_cms_config_cache else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}/cache/cached_cfg.txt"' + return ( + '' + if not enable_cms_config_cache + else 'kikimr_arg="${kikimr_arg} --cms-config-cache-file ${kikimr_home}/cache/cached_cfg.txt"' + ) def mbus_arguments(enable_mbus=False): - return [] if not enable_mbus else [ - 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ ${kikimr_mbus_enable}}"', - 'kikimr_arg="${kikimr_arg}${kikimr_mbus_port:+ --mbus-port ${kikimr_mbus_port}}"', - 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ --mbus-worker-count=4}"', - 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly:+ --mbus-max-in-flight ${kikimr_mbus_infly}}"', - 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly_bytes:+ --mbus-max-in-flight-by-size ${kikimr_mbus_infly_bytes}}"', - 'kikimr_arg="${kikimr_arg}${kikimr_mbus_max_message_size:+ --mbus-max-message-size ${kikimr_mbus_max_message_size}}"', - ] + return ( + [] + if not enable_mbus + else [ + 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ ${kikimr_mbus_enable}}"', + 'kikimr_arg="${kikimr_arg}${kikimr_mbus_port:+ --mbus-port ${kikimr_mbus_port}}"', + 'kikimr_arg="${kikimr_arg}${kikimr_mbus_enable:+ --mbus-worker-count=4}"', + 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly:+ --mbus-max-in-flight ${kikimr_mbus_infly}}"', + 'kikimr_arg="${kikimr_arg}${kikimr_mbus_infly_bytes:+ --mbus-max-in-flight-by-size ${kikimr_mbus_infly_bytes}}"', + 'kikimr_arg="${kikimr_arg}${kikimr_mbus_max_message_size:+ --mbus-max-message-size ${kikimr_mbus_max_message_size}}"', + ] + ) def dynamic_cfg_new_style( @@ -354,47 +387,53 @@ def kikimr_cfg_for_static_node_new_style( new_style_kikimr_cfg=True, mbus_enabled=False, ): - return "\n".join( [ local_vars( - tenant, ic_port=ic_port, + tenant, + ic_port=ic_port, mon_address=mon_address, - mon_port=mon_port, kikimr_home=kikimr_home, + mon_port=mon_port, + kikimr_home=kikimr_home, enable_cores=enable_cores, cert_params=cert_params, default_log_level=None, kikimr_binaries_base_path=None, new_style_kikimr_cfg=new_style_kikimr_cfg, - mbus_enabled=mbus_enabled + mbus_enabled=mbus_enabled, ), NEW_STYLE_CONFIG, - ] + mbus_arguments(mbus_enabled) + ] + + mbus_arguments(mbus_enabled) ) def kikimr_cfg_for_static_node( - tenant=None, - ic_port=19001, mon_port=8765, kikimr_home='/Berkanavt/kikimr', - pq_enable=False, - enable_cores=False, - default_log_level=3, - kikimr_binaries_base_path='/Berkanavt/kikimr', - mon_address="", - cert_params=None, - enable_cms_config_cache=False, - rb_txt_enabled=False, - metering_txt_enabled=False, - audit_txt_enabled=False, - yql_txt_enabled=False, - fq_txt_enabled=False, - mbus_enabled=False + tenant=None, + ic_port=19001, + mon_port=8765, + kikimr_home='/Berkanavt/kikimr', + pq_enable=False, + enable_cores=False, + default_log_level=3, + kikimr_binaries_base_path='/Berkanavt/kikimr', + mon_address="", + cert_params=None, + enable_cms_config_cache=False, + rb_txt_enabled=False, + metering_txt_enabled=False, + audit_txt_enabled=False, + yql_txt_enabled=False, + fq_txt_enabled=False, + mbus_enabled=False, ): return '\n'.join( [ local_vars( - tenant, ic_port=ic_port, - mon_port=mon_port, kikimr_home=kikimr_home, + tenant, + ic_port=ic_port, + mon_port=mon_port, + kikimr_home=kikimr_home, kikimr_binaries_base_path=kikimr_binaries_base_path, pq_enable=pq_enable, enable_cores=enable_cores, @@ -406,7 +445,7 @@ def kikimr_cfg_for_static_node( audit_txt_enabled=audit_txt_enabled, yql_txt_enabled=yql_txt_enabled, fq_txt_enabled=fq_txt_enabled, - mbus_enabled=mbus_enabled + mbus_enabled=mbus_enabled, ), NODE_ID_LOCAL_VAR, CUSTOM_CONFIG_INJECTOR, @@ -415,35 +454,43 @@ def kikimr_cfg_for_static_node( NODE_ID_ARGUMENT, tenant_argument(tenant), pq_enable_argument(pq_enable), - ] + rb_arguments(rb_txt_enabled) \ - + metering_arguments(metering_txt_enabled) \ - + audit_arguments(audit_txt_enabled) \ - + yql_arguments(yql_txt_enabled) \ - + ([cms_config_cache_argument_for_static_nodes(enable_cms_config_cache)] if enable_cms_config_cache else []) \ + ] + + rb_arguments(rb_txt_enabled) + + metering_arguments(metering_txt_enabled) + + audit_arguments(audit_txt_enabled) + + yql_arguments(yql_txt_enabled) + + ([cms_config_cache_argument_for_static_nodes(enable_cms_config_cache)] if enable_cms_config_cache else []) + mbus_arguments(mbus_enabled) ) def kikimr_cfg_for_dynamic_node( - node_broker_port=2135, tenant=None, - ic_port=19001, mon_port=8765, - kikimr_home='/Berkanavt/kikimr', sqs_port=8771, sqs_enable=False, - enable_cores=False, - default_log_level=3, - kikimr_binaries_base_path='/Berkanavt/kikimr', - mon_address="", cert_params=None, - enable_cms_config_cache=False, - rb_txt_enabled=False, - metering_txt_enabled=False, - audit_txt_enabled=False, - yql_txt_enabled=False, - fq_txt_enabled=False + node_broker_port=2135, + tenant=None, + ic_port=19001, + mon_port=8765, + kikimr_home='/Berkanavt/kikimr', + sqs_port=8771, + sqs_enable=False, + enable_cores=False, + default_log_level=3, + kikimr_binaries_base_path='/Berkanavt/kikimr', + mon_address="", + cert_params=None, + enable_cms_config_cache=False, + rb_txt_enabled=False, + metering_txt_enabled=False, + audit_txt_enabled=False, + yql_txt_enabled=False, + fq_txt_enabled=False, ): return "\n".join( [ local_vars( - tenant, node_broker_port=node_broker_port, - ic_port=ic_port, mon_port=mon_port, + tenant, + node_broker_port=node_broker_port, + ic_port=ic_port, + mon_port=mon_port, kikimr_home=kikimr_home, kikimr_binaries_base_path=kikimr_binaries_base_path, sqs_port=sqs_port, @@ -464,9 +511,10 @@ def kikimr_cfg_for_dynamic_node( NODE_BROKER_ARGUMENT, tenant_argument(tenant), sqs_arguments(sqs_enable), - ] + rb_arguments(rb_txt_enabled) \ - + metering_arguments(metering_txt_enabled) \ - + audit_arguments(audit_txt_enabled) \ + ] + + rb_arguments(rb_txt_enabled) + + metering_arguments(metering_txt_enabled) + + audit_arguments(audit_txt_enabled) + ([cms_config_cache_argument_for_dynamic_nodes(enable_cms_config_cache)] if enable_cms_config_cache else []) ) @@ -484,11 +532,7 @@ def expected_vars(**kwargs): kikimr_patched.extend(BASE_VARS) - return '\n'.join( - [ - '%s="%s"' % (var[0], var[1]) for var in kikimr_patched if var[0] not in exclude_options - ] - ) + return '\n'.join(['%s="%s"' % (var[0], var[1]) for var in kikimr_patched if var[0] not in exclude_options]) def kikimr_cfg_for_dynamic_slot( @@ -515,9 +559,11 @@ def kikimr_cfg_for_dynamic_slot( mon_port=None, home=None, syslog_service_tag=None, - exclude_options=['kikimr_system_file'] + exclude_options=['kikimr_system_file'], ), - ] + ['kikimr_%s=%s' % (key, value) for key, value in zip(['ca', 'cert', 'key'], cert_params or []) if value] + [ + ] + + ['kikimr_%s=%s' % (key, value) for key, value in zip(['ca', 'cert', 'key'], cert_params or []) if value] + + [ CUSTOM_CONFIG_INJECTOR, CUSTOM_SYS_CONFIG_INJECTOR, # arguments @@ -525,9 +571,10 @@ def kikimr_cfg_for_dynamic_slot( NODE_BROKER_ARGUMENT, SYS_LOG_SERVICE_TAG, tenant_argument(True), - ] + rb_arguments(rb_txt_enabled) \ - + metering_arguments(metering_txt_enabled) \ - + audit_arguments(audit_txt_enabled) \ - + yql_arguments(yql_txt_enabled) \ + ] + + rb_arguments(rb_txt_enabled) + + metering_arguments(metering_txt_enabled) + + audit_arguments(audit_txt_enabled) + + yql_arguments(yql_txt_enabled) + ([cms_config_cache_argument_for_dynamic_nodes(enable_cms_config_cache)] if enable_cms_config_cache else []) ) diff --git a/ydb/tools/cfg/types.py b/ydb/tools/cfg/types.py index 773f9cc70d..c401e7069d 100644 --- a/ydb/tools/cfg/types.py +++ b/ydb/tools/cfg/types.py @@ -24,14 +24,14 @@ def py3_ensure_bytes(value): @enum.unique class LogLevels(enum.IntEnum): - EMERG = 0, - ALERT = 1, - CRIT = 2, - ERROR = 3, - WARN = 4, - NOTICE = 5, - INFO = 6, - DEBUG = 7, + EMERG = (0,) + ALERT = (1,) + CRIT = (2,) + ERROR = (3,) + WARN = (4,) + NOTICE = (5,) + INFO = (6,) + DEBUG = (7,) TRACE = 8 @@ -48,7 +48,7 @@ class TabletTypes(enum.Enum): FLAT_TX_COORDINATOR = _tablet_type(13, 0x800001, service_name='TX_COORDINATOR') FLAT_HIVE = _tablet_type(14, 0xA001, is_unique=True, service_name='HIVE') FLAT_BS_CONTROLLER = _tablet_type(15, 0x1001, is_unique=True, service_name='BS_CONTROLLER') - FLAT_SCHEMESHARD = _tablet_type(16, 0x1000008587a0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD') + FLAT_SCHEMESHARD = _tablet_type(16, 0x1000008587A0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD') TX_ALLOCATOR = _tablet_type(23, 0x820001, service_name='TX_ALLOCATOR') FLAT_TX_PROXY = _tablet_type(17, 0x820001, service_name='TX_PROXY') FLAT_DATASHARD = _tablet_type(18, 999, service_name='DATASHARD') @@ -110,7 +110,7 @@ class TabletTypesFixed(enum.Enum): FLAT_TX_COORDINATOR = _tablet_type(13, 0x800001, service_name='TX_COORDINATOR') FLAT_HIVE = _tablet_type(14, 0xA001, is_unique=True, service_name='HIVE') FLAT_BS_CONTROLLER = _tablet_type(15, 0x1001, is_unique=True, service_name='BS_CONTROLLER') - FLAT_SCHEMESHARD = _tablet_type(16, 0x8587a0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD') + FLAT_SCHEMESHARD = _tablet_type(16, 0x8587A0, is_unique=True, service_name='FLAT_TX_SCHEMESHARD') TX_ALLOCATOR = _tablet_type(23, 0x820001, service_name='TX_ALLOCATOR') FLAT_TX_PROXY = _tablet_type(17, 0x820001, service_name='TX_PROXY') FLAT_DATASHARD = _tablet_type(18, 999, service_name='DATASHARD') diff --git a/ydb/tools/cfg/utils.py b/ydb/tools/cfg/utils.py index 98a285dbad..94c0533459 100644 --- a/ydb/tools/cfg/utils.py +++ b/ydb/tools/cfg/utils.py @@ -43,12 +43,7 @@ def write_to_file(file_path, value): def write_proto_to_file(file_path, proto): - write_to_file( - file_path, - message_to_string( - proto - ) - ) + write_to_file(file_path, message_to_string(proto)) def read_message_from_string(value, message): @@ -70,8 +65,7 @@ def get_resource_data(*args): def read_from_resource(message, *args): for key, value in resource.iteritems(get_resource_prefix(*args)): - return read_message_from_string( - value, message) + return read_message_from_string(value, message) raise RuntimeError() @@ -83,12 +77,7 @@ def get_resources_list(prefix=''): def capitalize_name(initial_name): - return ''.join( - map( - lambda name_part: name_part.capitalize(), - initial_name.split('_') - ) - ) + return ''.join(map(lambda name_part: name_part.capitalize(), initial_name.split('_'))) def to_lower(s): @@ -97,11 +86,7 @@ def to_lower(s): def get_canonical_name(name): chars = set(string.punctuation) - return to_lower( - "".join( - filter(lambda x: x not in chars, name) - ) - ) + return to_lower("".join(filter(lambda x: x not in chars, name))) def apply_config_changes(target, changes, fix_names=None): @@ -122,7 +107,10 @@ def apply_config_changes(target, changes, fix_names=None): if fixed_change_name is None: fixed_change_name = in_proto_field_names.get(get_canonical_name(change_name)) - assert fixed_change_name is not None, "Cannot find suitable field %s, proto descriptor: %s" % (change_name, target.DESCRIPTOR.full_name) + assert fixed_change_name is not None, "Cannot find suitable field %s, proto descriptor: %s" % ( + change_name, + target.DESCRIPTOR.full_name, + ) if isinstance(change_value, list): for item in change_value: @@ -150,7 +138,8 @@ def apply_config_changes(target, changes, fix_names=None): setattr(target, fixed_change_name, change_value) except Exception as e: raise RuntimeError( - "Cannot apply config change: change_name: %s, change_value %s. Reason: %s" % ( + "Cannot apply config change: change_name: %s, change_value %s. Reason: %s" + % ( change_name, str(change_value), str(e), diff --git a/ydb/tools/cfg/validation.py b/ydb/tools/cfg/validation.py index 2758d73814..c764e2b372 100644 --- a/ydb/tools/cfg/validation.py +++ b/ydb/tools/cfg/validation.py @@ -5,15 +5,12 @@ import copy import jsonschema from ydb.tools.cfg import utils -from ydb.tools.cfg.types import (Erasure, FailDomainType, LogLevels, NodeType, - PDiskCategory) +from ydb.tools.cfg.types import Erasure, FailDomainType, LogLevels, NodeType, PDiskCategory KQP_SCHEMA = { 'type': 'object', 'properties': { - 'enable': { - 'type': 'boolean' - }, + 'enable': {'type': 'boolean'}, }, } @@ -45,12 +42,12 @@ PROFILES = { 'type': 'string', }, }, - "additionalProperties": False + "additionalProperties": False, }, - "minItems": 3 + "minItems": 3, } - } - } + }, + }, } @@ -90,7 +87,7 @@ EXECUTOR_SCHEMA = { 'type': 'integer', 'min': 1, }, - } + }, } SYS_SCHEMA = { @@ -109,15 +106,15 @@ SYS_SCHEMA = { 'io': copy.deepcopy(EXECUTOR_SCHEMA), 'ic': copy.deepcopy(EXECUTOR_SCHEMA), }, - 'additionalProperties': False + 'additionalProperties': False, }, 'scheduler': { 'type': 'object', 'properties': { 'resolution': {'type': 'integer'}, 'spin_threshold': {'type': 'integer'}, - 'progress_threshold': {'type': 'integer'} - } + 'progress_threshold': {'type': 'integer'}, + }, }, 'use_auto_config': {'type': 'boolean'}, 'cpu_count': {'type': 'integer'}, @@ -126,7 +123,7 @@ SYS_SCHEMA = { 'enum': NodeType.all_node_type_names(), }, }, - 'additionalProperties': False + 'additionalProperties': False, } TRACING_SCHEMA = dict( @@ -143,15 +140,13 @@ TRACING_SCHEMA = dict( 'root_ca', 'service_name', ], - additionalProperties=False + additionalProperties=False, ) FAILURE_INJECTION_CONFIG_SCHEMA = { "type": "object", - "properties": { - 'approximate_termination_interval': dict(type='integer') - }, - "additionalProperties": False + "properties": {'approximate_termination_interval': dict(type='integer')}, + "additionalProperties": False, } DRIVE_SCHEMA = { @@ -161,30 +156,18 @@ DRIVE_SCHEMA = { 'path': dict(type='string', minLength=1), 'shared_with_os': dict(type='boolean'), 'expected_slot_count': dict(type='integer'), - 'kind': dict(type='integer') + 'kind': dict(type='integer'), }, - "required": [ - "type", - "path" - ], - "additionalProperties": False + "required": ["type", "path"], + "additionalProperties": False, } HOST_SCHEMA = { "type": "object", "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "drives": { - "type": "array", - "items": copy.deepcopy(DRIVE_SCHEMA) - }, - 'host_config_id': { - 'type': "integer", - 'minLength': 1 - }, + "name": {"type": "string", "minLength": 1}, + "drives": {"type": "array", "items": copy.deepcopy(DRIVE_SCHEMA)}, + 'host_config_id': {'type': "integer", 'minLength': 1}, "ic_port": { "type": "integer", }, @@ -197,29 +180,22 @@ HOST_SCHEMA = { LOG_SCHEMA = { 'type': "object", "properties": { - "default": { - "type": "integer", - "min": int(min(LogLevels)), - "max": int(max(LogLevels)) - }, + "default": {"type": "integer", "min": int(min(LogLevels)), "max": int(max(LogLevels))}, "entries": { "type": "array", "items": { "type": "object", "properties": { - "name": { - "type": "string", - "minLength": 1 - }, + "name": {"type": "string", "minLength": 1}, "level": { "type": "integer", "min": int(min(LogLevels)), "max": int(max(LogLevels)), - } - } - } - } - } + }, + }, + }, + }, + }, } STORAGE_POOL = { @@ -230,23 +206,13 @@ STORAGE_POOL = { 'minLength': 1, 'enum': Erasure.all_erasure_type_names(), }, - 'generation': { - 'type': 'integer', - 'min': 0 - }, - 'encryption_mode': { - 'type': 'integer', - 'min': 0, - 'max': 1 - }, + 'generation': {'type': 'integer', 'min': 0}, + 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1}, 'num_groups': { 'type': 'integer', 'min': 1, }, - 'storage_pool_id': { - 'type': 'integer', - 'min': 1 - }, + 'storage_pool_id': {'type': 'integer', 'min': 1}, "fail_domain_type": { "type": "string", "minLength": 1, @@ -273,7 +239,7 @@ STORAGE_POOL = { 'kind': dict(type='integer'), }, 'required': ['type'], - 'additionalProperties': False + 'additionalProperties': False, }, 'vdisk_kind': { 'type': 'string', @@ -300,11 +266,7 @@ STORAGE_POOL_KIND = { "minLength": 1, "enum": Erasure.all_erasure_type_names(), }, - 'encryption_mode': { - 'type': 'integer', - 'min': 0, - 'max': 1 - }, + 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1}, "filter_properties": { "type": "object", "properties": { @@ -318,7 +280,7 @@ STORAGE_POOL_KIND = { 'kind': dict(type='integer'), }, "required": ["type"], - "additionalProperties": False + "additionalProperties": False, }, "fail_domain_type": { "type": "string", @@ -330,11 +292,7 @@ STORAGE_POOL_KIND = { 'minLength': 1, }, }, - "required": [ - "kind", - "erasure", - "filter_properties" - ], + "required": ["kind", "erasure", "filter_properties"], } STORAGE_POOL_INSTANCE = { @@ -351,7 +309,7 @@ STORAGE_POOL_INSTANCE = { 'generation': { 'type': 'integer', 'min': 0, - } + }, }, 'required': [ 'kind', @@ -363,12 +321,8 @@ STORAGE_POOL_INSTANCE = { SHARED_CACHE_SCHEMA = { 'type': 'object', - 'properties': { - 'memory_limit': { - 'type': 'integer' - } - }, - 'additionalProperties': False + 'properties': {'memory_limit': {'type': 'integer'}}, + 'additionalProperties': False, } TENANT_SCHEMA = { @@ -386,12 +340,9 @@ TENANT_SCHEMA = { 'count': { 'type': 'integer', }, - 'kind': { - 'type': 'string', - 'minLength': 1 - } - } - } + 'kind': {'type': 'string', 'minLength': 1}, + }, + }, }, 'compute_units': { 'type': 'array', @@ -406,56 +357,35 @@ TENANT_SCHEMA = { 'type': 'string', 'minLength': 1, }, - 'zone': { - 'type': 'string', - 'minLength': 1 - }, + 'zone': {'type': 'string', 'minLength': 1}, 'required': ['count', 'kind', 'zone'], - 'additionalProperties': False - } - } + 'additionalProperties': False, + }, + }, }, 'overridden_configs': { 'type': 'object', - } - } + }, + }, } DOMAIN_SCHEMA = { 'type': 'object', 'properties': { - 'domain_name': { - 'type': 'string', - 'minLength': 1 - }, + 'domain_name': {'type': 'string', 'minLength': 1}, 'plan_resolution': { 'type': 'integer', 'min': 1, }, - 'domain_id': { - 'type': 'integer', - 'min': 1 - }, + 'domain_id': {'type': 'integer', 'min': 1}, 'mediators': { 'type': 'integer', 'min': 1, }, - 'coordinators': { - 'type': 'integer', - 'min': 1 - }, - 'allocators': { - 'type': 'integer', - 'min': 1 - }, - 'storage_pool_kinds': { - 'type': 'array', - 'items': copy.deepcopy(STORAGE_POOL_KIND) - }, - 'storage_pools': { - 'type': 'array', - 'items': copy.deepcopy(STORAGE_POOL_INSTANCE) - }, + 'coordinators': {'type': 'integer', 'min': 1}, + 'allocators': {'type': 'integer', 'min': 1}, + 'storage_pool_kinds': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_KIND)}, + 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_INSTANCE)}, 'databases': { 'type': 'array', 'items': copy.deepcopy(TENANT_SCHEMA), @@ -463,21 +393,18 @@ DOMAIN_SCHEMA = { 'config_cookie': { 'type': 'string', }, - 'dynamic_slots': { - 'type': 'integer', - 'min': 1 - }, + 'dynamic_slots': {'type': 'integer', 'min': 1}, 'bind_slots_to_numa_nodes': {'type': 'boolean'}, 'console_initializers': { 'type': 'array', 'items': { 'type': 'string', 'enum': utils.get_resources_list('resources/console_initializers/'), - } + }, }, }, 'required': ['domain_name'], - 'additionalProperties': False + 'additionalProperties': False, } NBS_SCHEMA = { @@ -489,11 +416,7 @@ NBS_SCHEMA = { 'log': copy.deepcopy(LOG_SCHEMA), 'domain': {'type': 'string'}, 'subdomain': {'type': 'string'}, - 'storage': { - 'type': 'object', - 'properties': { - } - }, + 'storage': {'type': 'object', 'properties': {}}, 'disk_registry_proxy': { 'type': 'object', }, @@ -522,47 +445,25 @@ NBS_SCHEMA = { }, }, }, - } - } - }, - 'discovery': { - 'type': 'object' - }, - 'ydbstats': { - 'type': 'object' - }, - 'auth': { - 'type': 'object' - }, - 'names': { - 'type': 'object' + }, + }, }, + 'discovery': {'type': 'object'}, + 'ydbstats': {'type': 'object'}, + 'auth': {'type': 'object'}, + 'names': {'type': 'object'}, 'client': { 'type': 'object', 'properties': { - 'auth_config': { - 'type': 'object' - }, - 'client_config': { - 'type': 'object' - }, - 'log_config': { - 'type': 'object' - }, - 'monitoring_config': { - 'type': 'object' - }, - 'throttling_enabled': { - 'type': 'boolean' - }, - 'throttling_enabled_s_s_d': { - 'type': 'boolean' - }, - 'throttling_disabled_s_s_d_nonrepl': { - 'type': 'boolean' - }, + 'auth_config': {'type': 'object'}, + 'client_config': {'type': 'object'}, + 'log_config': {'type': 'object'}, + 'monitoring_config': {'type': 'object'}, + 'throttling_enabled': {'type': 'boolean'}, + 'throttling_enabled_s_s_d': {'type': 'boolean'}, + 'throttling_disabled_s_s_d_nonrepl': {'type': 'boolean'}, }, - 'additionalProperties': False + 'additionalProperties': False, }, 'http_proxy': { 'type': 'object', @@ -603,7 +504,7 @@ NBS_SCHEMA = { 'type': 'boolean', }, }, - 'additionalProperties': False + 'additionalProperties': False, }, 'breakpad': { 'type': 'object', @@ -612,7 +513,7 @@ NBS_SCHEMA = { 'type': 'boolean', }, }, - 'additionalProperties': False + 'additionalProperties': False, }, 'breakpad_sender': { 'type': 'object', @@ -624,46 +525,23 @@ NBS_SCHEMA = { 'type': 'string', }, }, - 'additionalProperties': False + 'additionalProperties': False, }, 'logbroker': { 'type': 'object', 'properties': { - 'address': { - 'type': 'string' - }, - 'port': { - 'type': 'integer' - }, - 'database': { - 'type': 'string' - }, - 'use_logbroker_c_d_s': { - 'type': 'boolean' - }, - 'ca_cert_filename': { - 'type': 'string' - }, - 'topic': { - 'type': 'string' - }, - 'source_id': { - 'type': 'string' - }, - 'metadata_server_address': { - 'type': 'string' - } + 'address': {'type': 'string'}, + 'port': {'type': 'integer'}, + 'database': {'type': 'string'}, + 'use_logbroker_c_d_s': {'type': 'boolean'}, + 'ca_cert_filename': {'type': 'string'}, + 'topic': {'type': 'string'}, + 'source_id': {'type': 'string'}, + 'metadata_server_address': {'type': 'string'}, }, - 'additionalProperties': False - }, - 'notify': { - 'type': 'object', - 'properties': { - 'endpoint': { - 'type': 'string' - } - } + 'additionalProperties': False, }, + 'notify': {'type': 'object', 'properties': {'endpoint': {'type': 'string'}}}, "iam": { 'type': 'object', }, @@ -691,9 +569,9 @@ NBS_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - } + }, } - } + }, }, 'Blacklist': { 'type': 'object', @@ -702,18 +580,18 @@ NBS_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - } + }, } - } + }, }, }, }, }, }, - 'additionalProperties': False + 'additionalProperties': False, }, }, - 'additionalProperties': False + 'additionalProperties': False, } NFS_SCHEMA = { @@ -725,18 +603,10 @@ NFS_SCHEMA = { 'log': copy.deepcopy(LOG_SCHEMA), 'domain': {'type': 'string'}, 'subdomain': {'type': 'string'}, - 'names': { - 'type': 'object' - }, - 'storage': { - 'type': 'object' - }, - 'diagnostics': { - 'type': 'object' - }, - 'auth': { - 'type': 'object' - }, + 'names': {'type': 'object'}, + 'storage': {'type': 'object'}, + 'diagnostics': {'type': 'object'}, + 'auth': {'type': 'object'}, 'server': { 'type': 'object', 'properties': { @@ -745,8 +615,8 @@ NFS_SCHEMA = { }, 'port': { 'type': 'integer', - } - } + }, + }, }, 'vhost': { 'type': 'object', @@ -756,8 +626,8 @@ NFS_SCHEMA = { }, 'port': { 'type': 'integer', - } - } + }, + }, }, 'http_proxy': { 'type': 'object', @@ -772,10 +642,10 @@ NFS_SCHEMA = { 'type': 'integer', }, }, - 'additionalProperties': False + 'additionalProperties': False, }, }, - 'additionalProperties': False + 'additionalProperties': False, } SQS_SCHEMA = { @@ -787,8 +657,8 @@ SQS_SCHEMA = { 'subdomain': {'type': 'string'}, 'http_server': { 'type': 'object', - } - } + }, + }, } SOLOMON_SCHEMA = { @@ -805,10 +675,10 @@ SOLOMON_SCHEMA = { 'name': {'type': 'string'}, 'channels_profile': {'type': 'integer'}, 'partitions': {'type': 'integer'}, - } - } - } - } + }, + }, + }, + }, } CMS_LIMITS_SCHEMA = { @@ -816,7 +686,7 @@ CMS_LIMITS_SCHEMA = { 'properties': { 'disabled_nodes_limit': {'type': 'integer'}, 'disabled_nodes_ratio_limit': {'type': 'integer'}, - } + }, } CMS_SCHEMA = { @@ -837,9 +707,9 @@ CMS_SCHEMA = { 'broken_timeout_min': {'type': 'integer'}, 'broken_prep_timeout_min': {'type': 'integer'}, 'faulty_prep_timeout_min': {'type': 'integer'}, - } - } - } + }, + }, + }, } STATE_STORAGE = { @@ -847,21 +717,11 @@ STATE_STORAGE = { 'properties': { 'allow_incorrect': {'type': 'boolean'}, 'node_ids': {'type': 'array', 'items': {'type': 'integer'}}, - 'node_set': { - 'type': 'array', - 'items': { - 'type': 'string' - } - } - } + 'node_set': {'type': 'array', 'items': {'type': 'string'}}, + }, } -METERING_SCHEMA = { - 'type': 'object', - 'properties': { - 'metering_file': {'type': 'string'} - } -} +METERING_SCHEMA = {'type': 'object', 'properties': {'metering_file': {'type': 'string'}}} YQL_SCHEMA = { 'type': 'object', @@ -882,8 +742,8 @@ YQL_SCHEMA = { 'impersonation_service_endpoint': {'type': 'string'}, 'secure_task_service': {'type': 'boolean'}, 'secure_impersonation_service': {'type': 'boolean'}, - 'hmac_secret_file': {'type': 'string'} - } + 'hmac_secret_file': {'type': 'string'}, + }, } YQ_SCHEMA = { @@ -904,7 +764,7 @@ YQ_SCHEMA = { 'use_local_metadata_service': {'type': 'boolean'}, 'enable_forward_analytics': {'type': 'boolean'}, 'enable_permissions': {'type': 'boolean'}, - } + }, }, 'analytics': { 'type': 'object', @@ -926,8 +786,8 @@ YQ_SCHEMA = { 'impersonation_service_endpoint': {'type': 'string'}, 'secure_task_service': {'type': 'boolean'}, 'secure_impersonation_service': {'type': 'boolean'}, - 'hmac_secret_file': {'type': 'string'} - } + 'hmac_secret_file': {'type': 'string'}, + }, }, 'streaming': { 'type': 'object', @@ -947,27 +807,24 @@ YQ_SCHEMA = { 'properties': { 'enabled': {'type': 'boolean'}, 'period_millis': {'type': 'integer'}, - 'max_in_flight': {'type': 'integer'} - } - } - } + 'max_in_flight': {'type': 'integer'}, + }, + }, + }, }, 'token_accessor': { 'type': 'object', - 'properties': { - 'endpoint': {'type': 'string'}, - 'use_ssl': {'type': 'boolean'} - } + 'properties': {'endpoint': {'type': 'string'}, 'use_ssl': {'type': 'boolean'}}, }, 'folder_service': { 'type': 'object', 'properties': { 'enable': {'type': 'boolean'}, 'endpoint': {'type': 'string'}, - 'path_to_root_ca': {'type': 'string'} - } - } - } + 'path_to_root_ca': {'type': 'string'}, + }, + }, + }, } TEMPLATE_SCHEMA = { @@ -988,21 +845,19 @@ TEMPLATE_SCHEMA = { 'items': { 'type': 'object', 'properties': { - 'generation': { - 'type': 'integer', 'min': 0 - }, + 'generation': {'type': 'integer', 'min': 0}, 'drives': { 'type': 'array', 'items': copy.deepcopy(DRIVE_SCHEMA), - } - } - } + }, + }, + }, }, 'static_bs_group_hosts': { 'type': 'array', 'items': { 'type': 'string', - } + }, }, 'use_cluster_uuid': { 'type': 'boolean', @@ -1014,32 +869,16 @@ TEMPLATE_SCHEMA = { 'type': 'string', 'minLength': 1, }, - 'accepted_cluster_uuids': { - 'type': 'array', - 'items': { - 'type': 'string', - 'minLength': 1 - } - }, + 'accepted_cluster_uuids': {'type': 'array', 'items': {'type': 'string', 'minLength': 1}}, 'security_settings': { 'type': 'object', 'properties': { 'enforce_user_token_requirement': { 'type': 'boolean', }, - 'monitoring_allowed_sids': { - 'type': 'array', - 'items': { - 'type': 'string' - } - }, - 'administration_allowed_sids': { - 'type': 'array', - 'items': { - 'type': 'string' - } - }, - } + 'monitoring_allowed_sids': {'type': 'array', 'items': {'type': 'string'}}, + 'administration_allowed_sids': {'type': 'array', 'items': {'type': 'string'}}, + }, }, "static_erasure": { "type": "string", @@ -1067,24 +906,15 @@ TEMPLATE_SCHEMA = { 'minItems': 1, 'checkNameServiceDuplicates': True, }, - 'auth': { - 'type': 'object' - }, + 'auth': {'type': 'object'}, 'log': copy.deepcopy(LOG_SCHEMA), 'grpc': {'type': 'object'}, 'kqp': copy.deepcopy(KQP_SCHEMA), 'ic': {'type': 'object'}, 'pq': {'type': 'object'}, - 'storage_pools': { - 'type': 'array', - 'items': copy.deepcopy(STORAGE_POOL) - }, + 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL)}, 'profiles': copy.deepcopy(PROFILES), - 'domains': { - 'type': 'array', - 'items': copy.deepcopy(DOMAIN_SCHEMA), - 'minItems': 1 - }, + 'domains': {'type': 'array', 'items': copy.deepcopy(DOMAIN_SCHEMA), 'minItems': 1}, 'nbs': copy.deepcopy(NBS_SCHEMA), 'nbs_control': copy.deepcopy(NBS_SCHEMA), 'nfs': copy.deepcopy(NFS_SCHEMA), @@ -1100,43 +930,28 @@ TEMPLATE_SCHEMA = { 'resource_broker': {'type': 'object'}, 'state_storages': { 'type': 'array', - 'items': { - 'type': 'object', - 'properties': {}, - 'additionalProperties': True - } + 'items': {'type': 'object', 'properties': {}, 'additionalProperties': True}, }, 'metering': copy.deepcopy(METERING_SCHEMA), 'yql_analytics': copy.deepcopy(YQL_SCHEMA), - 'yq': copy.deepcopy(YQ_SCHEMA) + 'yq': copy.deepcopy(YQ_SCHEMA), }, - "required": [ - "static_erasure", - "hosts" - ], + "required": ["static_erasure", "hosts"], } def _host_and_ic_port(host): - return "%s:%s" % ( - host['name'], str( - host.get('ic_port', 19001) - ) - ) + return "%s:%s" % (host['name'], str(host.get('ic_port', 19001))) def checkNameServiceDuplicates(validator, allow_duplicates, instance, schema): - names = collections.Counter( - [ - _host_and_ic_port(host) - for host in instance - ] - ) + names = collections.Counter([_host_and_ic_port(host) for host in instance]) for name, count in names.items(): if count > 1: yield jsonschema.ValidationError( - "Names of items contains non-unique elements %r: %s. " % ( + "Names of items contains non-unique elements %r: %s. " + % ( instance, name, ) @@ -1145,20 +960,17 @@ def checkNameServiceDuplicates(validator, allow_duplicates, instance, schema): _Validator = jsonschema.Draft4Validator _Validator = jsonschema.validators.extend( - _Validator, { + _Validator, + { 'checkNameServiceDuplicates': checkNameServiceDuplicates, - } + }, ) class Validator(_Validator): - def __init__(self, schema): format_checker = jsonschema.FormatChecker() - super(Validator, self).__init__( - schema, - format_checker=format_checker - ) + super(Validator, self).__init__(schema, format_checker=format_checker) def validate(template): diff --git a/ydb/tools/cfg/walle/__init__.py b/ydb/tools/cfg/walle/__init__.py index 58cd44787a..df7d52021c 100644 --- a/ydb/tools/cfg/walle/__init__.py +++ b/ydb/tools/cfg/walle/__init__.py @@ -2,6 +2,4 @@ # -*- coding: utf-8 -*- from .walle import HostsFetcherInterface, NopHostsFetcher # noqa -__all__ = ( - 'Walle', -) +__all__ = ('Walle',) diff --git a/ydb/tools/cfg/walle/walle.py b/ydb/tools/cfg/walle/walle.py index 82a5cb70e3..92792aaf6e 100644 --- a/ydb/tools/cfg/walle/walle.py +++ b/ydb/tools/cfg/walle/walle.py @@ -4,7 +4,7 @@ import zlib from abc import ABCMeta, abstractmethod -class HostsFetcherInterface(): +class HostsFetcherInterface: __metaclass__ = ABCMeta @abstractmethod |