aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorshmel1k <shmel1k@ydb.tech>2023-11-28 17:59:19 +0300
committershmel1k <shmel1k@ydb.tech>2023-11-28 19:21:01 +0300
commitfc839da62d0e112f26540ebebaf964d93890f2fb (patch)
tree31d751464c357aa97cfbea4ea70cf315480b9ab7
parent10ae15c98d33118e47701756828019e97aa10ec6 (diff)
downloadydb-fc839da62d0e112f26540ebebaf964d93890f2fb.tar.gz
add Walle opensource adapter
-rw-r--r--ydb/tools/cfg/base.py2
-rw-r--r--ydb/tools/cfg/bin/__main__.py35
-rw-r--r--ydb/tools/cfg/validation.py1213
-rw-r--r--ydb/tools/cfg/walle/__init__.py4
-rw-r--r--ydb/tools/cfg/walle/walle.py75
5 files changed, 704 insertions, 625 deletions
diff --git a/ydb/tools/cfg/base.py b/ydb/tools/cfg/base.py
index 3bf9c8a0a8..63d1cbdf97 100644
--- a/ydb/tools/cfg/base.py
+++ b/ydb/tools/cfg/base.py
@@ -266,7 +266,7 @@ def normalize_domain(domain_name):
class ClusterDetailsProvider(object):
def __init__(self, template, walle_provider, validator=None, database=None):
if not validator:
- validator = validation.Validator()
+ validator = validation.default_validator()
self.__validator = validator
diff --git a/ydb/tools/cfg/bin/__main__.py b/ydb/tools/cfg/bin/__main__.py
index a4c6156311..c76379907f 100644
--- a/ydb/tools/cfg/bin/__main__.py
+++ b/ydb/tools/cfg/bin/__main__.py
@@ -11,24 +11,25 @@ from ydb.tools.cfg.configurator_setup import get_parser, parse_optional_argument
from ydb.tools.cfg.dynamic import DynamicConfigGenerator
from ydb.tools.cfg.static import StaticConfigGenerator
from ydb.tools.cfg.utils import write_to_file
+from ydb.tools.cfg.walle import NopHostsInformationProvider, WalleHostsInformationProvider
logging_config.dictConfig(
{
- 'version': 1,
- 'formatters': {
- 'base': {
- 'format': '%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s',
+ "version": 1,
+ "formatters": {
+ "base": {
+ "format": "%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s",
},
},
- 'handlers': {
- 'console': {
- 'class': 'logging.StreamHandler',
- 'level': 'INFO',
- 'formatter': 'base',
- 'stream': sys.stdout,
+ "handlers": {
+ "console": {
+ "class": "logging.StreamHandler",
+ "level": "INFO",
+ "formatter": "base",
+ "stream": sys.stdout,
},
},
- 'root': {'formatter': 'base', 'level': 'INFO', 'handlers': ('console',)},
+ "root": {"formatter": "base", "level": "INFO", "handlers": ("console",)},
}
)
@@ -44,10 +45,14 @@ def cfg_generate(args):
else:
cfg_cls = StaticConfigGenerator
- with open(args.cluster_description, 'r') as yaml_template:
+ with open(args.cluster_description, "r") as yaml_template:
cluster_template = yaml.safe_load(yaml_template)
- generator = cfg_cls(cluster_template, args.binary_path, args.output_dir, **kwargs)
+ hosts_provider = NopHostsInformationProvider()
+ if args.hosts_provider_url:
+ hosts_provider = WalleHostsInformationProvider(args.hosts_provider_url)
+
+ generator = cfg_cls(cluster_template, args.binary_path, args.output_dir, walle_provider=hosts_provider, **kwargs)
all_configs = generator.get_all_configs()
for cfg_name, cfg_value in all_configs.items():
@@ -55,10 +60,10 @@ def cfg_generate(args):
def main():
- parser = get_parser(cfg_generate)
+ parser = get_parser(cfg_generate, [{"name": "--hosts-provider-url", "help": "URL from which information about hosts can be obtained."}])
args = parser.parse_args()
args.func(args)
-if __name__ == '__main__':
+if __name__ == "__main__":
exit(main())
diff --git a/ydb/tools/cfg/validation.py b/ydb/tools/cfg/validation.py
index c764e2b372..8923291264 100644
--- a/ydb/tools/cfg/validation.py
+++ b/ydb/tools/cfg/validation.py
@@ -8,9 +8,9 @@ from ydb.tools.cfg import utils
from ydb.tools.cfg.types import Erasure, FailDomainType, LogLevels, NodeType, PDiskCategory
KQP_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
},
}
@@ -18,28 +18,28 @@ KQP_SCHEMA = {
# These profiles will be on top of channels.txt file so
# user can influence on id of these profiles
PROFILES = {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'channels': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'pdisk_type': {
- 'type': 'string',
- 'enum': PDiskCategory.all_pdisk_category_names(),
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "channels": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "pdisk_type": {
+ "type": "string",
+ "enum": PDiskCategory.all_pdisk_category_names(),
},
- 'erasure': {
- 'type': 'string',
- 'enum': Erasure.all_erasure_type_names(),
+ "erasure": {
+ "type": "string",
+ "enum": Erasure.all_erasure_type_names(),
},
- 'vdisk_kind': {
- 'type': 'string',
+ "vdisk_kind": {
+ "type": "string",
},
- 'storage_pool_kind': {
- 'type': 'string',
+ "storage_pool_kind": {
+ "type": "string",
},
},
"additionalProperties": False,
@@ -52,111 +52,111 @@ PROFILES = {
FEATURES_SCHEMA = {
- 'type': 'object',
- 'properties': {},
+ "type": "object",
+ "properties": {},
}
EXECUTOR_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'threads': {
- 'type': "integer",
- 'min': 1,
- },
- 'spin_threshold': {
- "type": 'integer',
+ "type": "object",
+ "properties": {
+ "threads": {
+ "type": "integer",
"min": 1,
},
- 'TimePerMailboxMicroSecs': {
- 'type': 'integer',
+ "spin_threshold": {
+ "type": "integer",
"min": 1,
},
- 'time_per_mailbox_micro_secs': {
- 'type': 'integer',
+ "TimePerMailboxMicroSecs": {
+ "type": "integer",
+ "min": 1,
+ },
+ "time_per_mailbox_micro_secs": {
+ "type": "integer",
"min": 1,
},
- 'max_threads': {
- 'type': 'integer',
- 'min': 1,
+ "max_threads": {
+ "type": "integer",
+ "min": 1,
},
- 'priority': {
- 'type': 'integer',
- 'min': 1,
+ "priority": {
+ "type": "integer",
+ "min": 1,
},
- 'max_avg_ping_deviation': {
- 'type': 'integer',
- 'min': 1,
+ "max_avg_ping_deviation": {
+ "type": "integer",
+ "min": 1,
},
},
}
SYS_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'preset_name': {
- 'type': 'string',
- 'enum': utils.get_resources_list('resources/sys/'),
- },
- 'executors': {
- 'type': 'object',
- 'properties': {
- 'system': copy.deepcopy(EXECUTOR_SCHEMA),
- 'batch': copy.deepcopy(EXECUTOR_SCHEMA),
- 'user': copy.deepcopy(EXECUTOR_SCHEMA),
- 'io': copy.deepcopy(EXECUTOR_SCHEMA),
- 'ic': copy.deepcopy(EXECUTOR_SCHEMA),
+ "type": "object",
+ "properties": {
+ "preset_name": {
+ "type": "string",
+ "enum": utils.get_resources_list("resources/sys/"),
+ },
+ "executors": {
+ "type": "object",
+ "properties": {
+ "system": copy.deepcopy(EXECUTOR_SCHEMA),
+ "batch": copy.deepcopy(EXECUTOR_SCHEMA),
+ "user": copy.deepcopy(EXECUTOR_SCHEMA),
+ "io": copy.deepcopy(EXECUTOR_SCHEMA),
+ "ic": copy.deepcopy(EXECUTOR_SCHEMA),
},
- 'additionalProperties': False,
- },
- 'scheduler': {
- 'type': 'object',
- 'properties': {
- 'resolution': {'type': 'integer'},
- 'spin_threshold': {'type': 'integer'},
- 'progress_threshold': {'type': 'integer'},
+ "additionalProperties": False,
+ },
+ "scheduler": {
+ "type": "object",
+ "properties": {
+ "resolution": {"type": "integer"},
+ "spin_threshold": {"type": "integer"},
+ "progress_threshold": {"type": "integer"},
},
},
- 'use_auto_config': {'type': 'boolean'},
- 'cpu_count': {'type': 'integer'},
- 'node_type': {
- 'type': 'string',
- 'enum': NodeType.all_node_type_names(),
+ "use_auto_config": {"type": "boolean"},
+ "cpu_count": {"type": "integer"},
+ "node_type": {
+ "type": "string",
+ "enum": NodeType.all_node_type_names(),
},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
}
TRACING_SCHEMA = dict(
- type='object',
+ type="object",
properties=dict(
- host=dict(type='string'),
- port=dict(type='integer'),
- root_ca=dict(type='string'),
- service_name=dict(type='string'),
+ host=dict(type="string"),
+ port=dict(type="integer"),
+ root_ca=dict(type="string"),
+ service_name=dict(type="string"),
),
required=[
- 'host',
- 'port',
- 'root_ca',
- 'service_name',
+ "host",
+ "port",
+ "root_ca",
+ "service_name",
],
additionalProperties=False,
)
FAILURE_INJECTION_CONFIG_SCHEMA = {
"type": "object",
- "properties": {'approximate_termination_interval': dict(type='integer')},
+ "properties": {"approximate_termination_interval": dict(type="integer")},
"additionalProperties": False,
}
DRIVE_SCHEMA = {
"type": "object",
"properties": {
- 'type': dict(type='string', enum=PDiskCategory.all_pdisk_category_names()),
- 'path': dict(type='string', minLength=1),
- 'shared_with_os': dict(type='boolean'),
- 'expected_slot_count': dict(type='integer'),
- 'kind': dict(type='integer'),
+ "type": dict(type="string", enum=PDiskCategory.all_pdisk_category_names()),
+ "path": dict(type="string", minLength=1),
+ "shared_with_os": dict(type="boolean"),
+ "expected_slot_count": dict(type="integer"),
+ "kind": dict(type="integer"),
},
"required": ["type", "path"],
"additionalProperties": False,
@@ -167,7 +167,7 @@ HOST_SCHEMA = {
"properties": {
"name": {"type": "string", "minLength": 1},
"drives": {"type": "array", "items": copy.deepcopy(DRIVE_SCHEMA)},
- 'host_config_id': {'type': "integer", 'minLength': 1},
+ "host_config_id": {"type": "integer", "minLength": 1},
"ic_port": {
"type": "integer",
},
@@ -178,7 +178,7 @@ HOST_SCHEMA = {
}
LOG_SCHEMA = {
- 'type': "object",
+ "type": "object",
"properties": {
"default": {"type": "integer", "min": int(min(LogLevels)), "max": int(max(LogLevels))},
"entries": {
@@ -199,59 +199,59 @@ LOG_SCHEMA = {
}
STORAGE_POOL = {
- 'type': 'object',
- 'properties': {
- 'erasure': {
- 'type': 'string',
- 'minLength': 1,
- 'enum': Erasure.all_erasure_type_names(),
- },
- 'generation': {'type': 'integer', 'min': 0},
- 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1},
- 'num_groups': {
- 'type': 'integer',
- 'min': 1,
- },
- 'storage_pool_id': {'type': 'integer', 'min': 1},
+ "type": "object",
+ "properties": {
+ "erasure": {
+ "type": "string",
+ "minLength": 1,
+ "enum": Erasure.all_erasure_type_names(),
+ },
+ "generation": {"type": "integer", "min": 0},
+ "encryption_mode": {"type": "integer", "min": 0, "max": 1},
+ "num_groups": {
+ "type": "integer",
+ "min": 1,
+ },
+ "storage_pool_id": {"type": "integer", "min": 1},
"fail_domain_type": {
"type": "string",
"minLength": 1,
"enum": FailDomainType.all_fail_domain_type_names(),
},
- 'kind': {
- 'type': 'string',
- 'minLength': 1,
+ "kind": {
+ "type": "string",
+ "minLength": 1,
},
- 'name': {
- 'type': 'string',
- 'minLength': 1,
+ "name": {
+ "type": "string",
+ "minLength": 1,
},
- 'filter_properties': {
- 'type': 'object',
- 'properties': {
- 'type': {
- 'type': 'string',
- 'enum': PDiskCategory.all_pdisk_category_names(),
+ "filter_properties": {
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": PDiskCategory.all_pdisk_category_names(),
},
- 'SharedWithOs': {
- 'type': 'boolean',
+ "SharedWithOs": {
+ "type": "boolean",
},
- 'kind': dict(type='integer'),
+ "kind": dict(type="integer"),
},
- 'required': ['type'],
- 'additionalProperties': False,
+ "required": ["type"],
+ "additionalProperties": False,
},
- 'vdisk_kind': {
- 'type': 'string',
- 'minLength': 1,
+ "vdisk_kind": {
+ "type": "string",
+ "minLength": 1,
},
},
- 'required': [
- 'num_groups',
- 'erasure',
- 'filter_properties',
+ "required": [
+ "num_groups",
+ "erasure",
+ "filter_properties",
],
- 'additionalProperties': False,
+ "additionalProperties": False,
}
STORAGE_POOL_KIND = {
@@ -259,14 +259,14 @@ STORAGE_POOL_KIND = {
"properties": {
"kind": {
"type": "string",
- 'minLength': 1,
+ "minLength": 1,
},
"erasure": {
"type": "string",
"minLength": 1,
"enum": Erasure.all_erasure_type_names(),
},
- 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1},
+ "encryption_mode": {"type": "integer", "min": 0, "max": 1},
"filter_properties": {
"type": "object",
"properties": {
@@ -274,10 +274,10 @@ STORAGE_POOL_KIND = {
"type": "string",
"enum": PDiskCategory.all_pdisk_category_names(),
},
- 'SharedWithOs': {
- 'type': 'boolean',
+ "SharedWithOs": {
+ "type": "boolean",
},
- 'kind': dict(type='integer'),
+ "kind": dict(type="integer"),
},
"required": ["type"],
"additionalProperties": False,
@@ -287,299 +287,299 @@ STORAGE_POOL_KIND = {
"minLength": 1,
"enum": FailDomainType.all_fail_domain_type_names(),
},
- 'vdisk_kind': {
- 'type': 'string',
- 'minLength': 1,
+ "vdisk_kind": {
+ "type": "string",
+ "minLength": 1,
},
},
"required": ["kind", "erasure", "filter_properties"],
}
STORAGE_POOL_INSTANCE = {
- 'type': 'object',
- 'properties': {
- 'kind': {
- 'type': 'string',
- 'minLength': 1,
+ "type": "object",
+ "properties": {
+ "kind": {
+ "type": "string",
+ "minLength": 1,
},
- 'num_groups': {
- 'type': 'integer',
- 'min': 1,
+ "num_groups": {
+ "type": "integer",
+ "min": 1,
},
- 'generation': {
- 'type': 'integer',
- 'min': 0,
+ "generation": {
+ "type": "integer",
+ "min": 0,
},
},
- 'required': [
- 'kind',
- 'num_groups',
+ "required": [
+ "kind",
+ "num_groups",
],
- 'additionalProperties': False,
+ "additionalProperties": False,
}
SHARED_CACHE_SCHEMA = {
- 'type': 'object',
- 'properties': {'memory_limit': {'type': 'integer'}},
- 'additionalProperties': False,
+ "type": "object",
+ "properties": {"memory_limit": {"type": "integer"}},
+ "additionalProperties": False,
}
TENANT_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'name': {
- 'type': 'string',
- 'minLength': 1,
- },
- 'storage_units': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'count': {
- 'type': 'integer',
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ },
+ "storage_units": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "count": {
+ "type": "integer",
},
- 'kind': {'type': 'string', 'minLength': 1},
+ "kind": {"type": "string", "minLength": 1},
},
},
},
- 'compute_units': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'count': {
- 'type': 'integer',
- 'min': 1,
+ "compute_units": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "count": {
+ "type": "integer",
+ "min": 1,
},
- 'kind': {
- 'type': 'string',
- 'minLength': 1,
+ "kind": {
+ "type": "string",
+ "minLength": 1,
},
- 'zone': {'type': 'string', 'minLength': 1},
- 'required': ['count', 'kind', 'zone'],
- 'additionalProperties': False,
+ "zone": {"type": "string", "minLength": 1},
+ "required": ["count", "kind", "zone"],
+ "additionalProperties": False,
},
},
},
- 'overridden_configs': {
- 'type': 'object',
+ "overridden_configs": {
+ "type": "object",
},
},
}
DOMAIN_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'domain_name': {'type': 'string', 'minLength': 1},
- 'plan_resolution': {
- 'type': 'integer',
- 'min': 1,
- },
- 'domain_id': {'type': 'integer', 'min': 1},
- 'mediators': {
- 'type': 'integer',
- 'min': 1,
- },
- 'coordinators': {'type': 'integer', 'min': 1},
- 'allocators': {'type': 'integer', 'min': 1},
- 'storage_pool_kinds': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_KIND)},
- 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_INSTANCE)},
- 'databases': {
- 'type': 'array',
- 'items': copy.deepcopy(TENANT_SCHEMA),
- },
- 'config_cookie': {
- 'type': 'string',
- },
- 'dynamic_slots': {'type': 'integer', 'min': 1},
- 'bind_slots_to_numa_nodes': {'type': 'boolean'},
- 'console_initializers': {
- 'type': 'array',
- 'items': {
- 'type': 'string',
- 'enum': utils.get_resources_list('resources/console_initializers/'),
+ "type": "object",
+ "properties": {
+ "domain_name": {"type": "string", "minLength": 1},
+ "plan_resolution": {
+ "type": "integer",
+ "min": 1,
+ },
+ "domain_id": {"type": "integer", "min": 1},
+ "mediators": {
+ "type": "integer",
+ "min": 1,
+ },
+ "coordinators": {"type": "integer", "min": 1},
+ "allocators": {"type": "integer", "min": 1},
+ "storage_pool_kinds": {"type": "array", "items": copy.deepcopy(STORAGE_POOL_KIND)},
+ "storage_pools": {"type": "array", "items": copy.deepcopy(STORAGE_POOL_INSTANCE)},
+ "databases": {
+ "type": "array",
+ "items": copy.deepcopy(TENANT_SCHEMA),
+ },
+ "config_cookie": {
+ "type": "string",
+ },
+ "dynamic_slots": {"type": "integer", "min": 1},
+ "bind_slots_to_numa_nodes": {"type": "boolean"},
+ "console_initializers": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": utils.get_resources_list("resources/console_initializers/"),
},
},
},
- 'required': ['domain_name'],
- 'additionalProperties': False,
+ "required": ["domain_name"],
+ "additionalProperties": False,
}
NBS_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'new_config_generator_enabled': {'type': 'boolean'},
- 'sys': copy.deepcopy(SYS_SCHEMA),
- 'log': copy.deepcopy(LOG_SCHEMA),
- 'domain': {'type': 'string'},
- 'subdomain': {'type': 'string'},
- 'storage': {'type': 'object', 'properties': {}},
- 'disk_registry_proxy': {
- 'type': 'object',
- },
- 'server': {
- 'type': 'object',
- 'properties': {
- 'host': {
- 'type': 'string',
- },
- 'port': {
- 'type': 'integer',
- },
- 'root_certs_file': {
- 'type': 'string',
- },
- 'certs': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'cert_file': {
- 'type': 'string',
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "new_config_generator_enabled": {"type": "boolean"},
+ "sys": copy.deepcopy(SYS_SCHEMA),
+ "log": copy.deepcopy(LOG_SCHEMA),
+ "domain": {"type": "string"},
+ "subdomain": {"type": "string"},
+ "storage": {"type": "object", "properties": {}},
+ "disk_registry_proxy": {
+ "type": "object",
+ },
+ "server": {
+ "type": "object",
+ "properties": {
+ "host": {
+ "type": "string",
+ },
+ "port": {
+ "type": "integer",
+ },
+ "root_certs_file": {
+ "type": "string",
+ },
+ "certs": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "cert_file": {
+ "type": "string",
},
- 'cert_private_key_file': {
- 'type': 'string',
+ "cert_private_key_file": {
+ "type": "string",
},
},
},
},
},
},
- 'discovery': {'type': 'object'},
- 'ydbstats': {'type': 'object'},
- 'auth': {'type': 'object'},
- 'names': {'type': 'object'},
- 'client': {
- 'type': 'object',
- 'properties': {
- 'auth_config': {'type': 'object'},
- 'client_config': {'type': 'object'},
- 'log_config': {'type': 'object'},
- 'monitoring_config': {'type': 'object'},
- 'throttling_enabled': {'type': 'boolean'},
- 'throttling_enabled_s_s_d': {'type': 'boolean'},
- 'throttling_disabled_s_s_d_nonrepl': {'type': 'boolean'},
+ "discovery": {"type": "object"},
+ "ydbstats": {"type": "object"},
+ "auth": {"type": "object"},
+ "names": {"type": "object"},
+ "client": {
+ "type": "object",
+ "properties": {
+ "auth_config": {"type": "object"},
+ "client_config": {"type": "object"},
+ "log_config": {"type": "object"},
+ "monitoring_config": {"type": "object"},
+ "throttling_enabled": {"type": "boolean"},
+ "throttling_enabled_s_s_d": {"type": "boolean"},
+ "throttling_disabled_s_s_d_nonrepl": {"type": "boolean"},
},
- 'additionalProperties': False,
- },
- 'http_proxy': {
- 'type': 'object',
- 'properties': {
- 'port': {
- 'type': 'integer',
- },
- 'secure_port': {
- 'type': 'integer',
- },
- 'certs': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'cert_file': {
- 'type': 'string',
+ "additionalProperties": False,
+ },
+ "http_proxy": {
+ "type": "object",
+ "properties": {
+ "port": {
+ "type": "integer",
+ },
+ "secure_port": {
+ "type": "integer",
+ },
+ "certs": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "cert_file": {
+ "type": "string",
},
- 'cert_private_key_file': {
- 'type': 'string',
+ "cert_private_key_file": {
+ "type": "string",
},
},
},
},
- 'nbs_server_host': {
- 'type': 'string',
+ "nbs_server_host": {
+ "type": "string",
},
- 'nbs_server_port': {
- 'type': 'integer',
+ "nbs_server_port": {
+ "type": "integer",
},
- 'nbs_server_cert_file': {
- 'type': 'string',
+ "nbs_server_cert_file": {
+ "type": "string",
},
- 'root_certs_file': {
- 'type': 'string',
+ "root_certs_file": {
+ "type": "string",
},
- 'nbs_server_insecure': {
- 'type': 'boolean',
+ "nbs_server_insecure": {
+ "type": "boolean",
},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
},
- 'breakpad': {
- 'type': 'object',
- 'properties': {
- 'enable': {
- 'type': 'boolean',
+ "breakpad": {
+ "type": "object",
+ "properties": {
+ "enable": {
+ "type": "boolean",
},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
},
- 'breakpad_sender': {
- 'type': 'object',
- 'properties': {
- 'aggregator_url': {
- 'type': 'string',
+ "breakpad_sender": {
+ "type": "object",
+ "properties": {
+ "aggregator_url": {
+ "type": "string",
},
- 'notify_email': {
- 'type': 'string',
+ "notify_email": {
+ "type": "string",
},
},
- 'additionalProperties': False,
- },
- 'logbroker': {
- 'type': 'object',
- 'properties': {
- 'address': {'type': 'string'},
- 'port': {'type': 'integer'},
- 'database': {'type': 'string'},
- 'use_logbroker_c_d_s': {'type': 'boolean'},
- 'ca_cert_filename': {'type': 'string'},
- 'topic': {'type': 'string'},
- 'source_id': {'type': 'string'},
- 'metadata_server_address': {'type': 'string'},
+ "additionalProperties": False,
+ },
+ "logbroker": {
+ "type": "object",
+ "properties": {
+ "address": {"type": "string"},
+ "port": {"type": "integer"},
+ "database": {"type": "string"},
+ "use_logbroker_c_d_s": {"type": "boolean"},
+ "ca_cert_filename": {"type": "string"},
+ "topic": {"type": "string"},
+ "source_id": {"type": "string"},
+ "metadata_server_address": {"type": "string"},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
},
- 'notify': {'type': 'object', 'properties': {'endpoint': {'type': 'string'}}},
+ "notify": {"type": "object", "properties": {"endpoint": {"type": "string"}}},
"iam": {
- 'type': 'object',
+ "type": "object",
},
"kms": {
- 'type': 'object',
+ "type": "object",
},
"compute": {
- 'type': 'object',
- },
- 'features': {
- 'type': 'object',
- 'properties': {
- 'features': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'name': {
- 'type': 'string',
+ "type": "object",
+ },
+ "features": {
+ "type": "object",
+ "properties": {
+ "features": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string",
},
- 'Whitelist': {
- 'type': 'object',
- 'properties': {
- 'cloud_ids': {
- 'type': 'array',
- 'items': {
- 'type': 'string',
+ "Whitelist": {
+ "type": "object",
+ "properties": {
+ "cloud_ids": {
+ "type": "array",
+ "items": {
+ "type": "string",
},
}
},
},
- 'Blacklist': {
- 'type': 'object',
- 'properties': {
- 'cloud_ids': {
- 'type': 'array',
- 'items': {
- 'type': 'string',
+ "Blacklist": {
+ "type": "object",
+ "properties": {
+ "cloud_ids": {
+ "type": "array",
+ "items": {
+ "type": "string",
},
}
},
@@ -588,93 +588,93 @@ NBS_SCHEMA = {
},
},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
}
NFS_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'new_config_generator_enabled': {'type': 'boolean'},
- 'sys': copy.deepcopy(SYS_SCHEMA),
- 'log': copy.deepcopy(LOG_SCHEMA),
- 'domain': {'type': 'string'},
- 'subdomain': {'type': 'string'},
- 'names': {'type': 'object'},
- 'storage': {'type': 'object'},
- 'diagnostics': {'type': 'object'},
- 'auth': {'type': 'object'},
- 'server': {
- 'type': 'object',
- 'properties': {
- 'host': {
- 'type': 'string',
- },
- 'port': {
- 'type': 'integer',
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "new_config_generator_enabled": {"type": "boolean"},
+ "sys": copy.deepcopy(SYS_SCHEMA),
+ "log": copy.deepcopy(LOG_SCHEMA),
+ "domain": {"type": "string"},
+ "subdomain": {"type": "string"},
+ "names": {"type": "object"},
+ "storage": {"type": "object"},
+ "diagnostics": {"type": "object"},
+ "auth": {"type": "object"},
+ "server": {
+ "type": "object",
+ "properties": {
+ "host": {
+ "type": "string",
+ },
+ "port": {
+ "type": "integer",
},
},
},
- 'vhost': {
- 'type': 'object',
- 'properties': {
- 'host': {
- 'type': 'string',
+ "vhost": {
+ "type": "object",
+ "properties": {
+ "host": {
+ "type": "string",
},
- 'port': {
- 'type': 'integer',
+ "port": {
+ "type": "integer",
},
},
},
- 'http_proxy': {
- 'type': 'object',
- 'properties': {
- 'port': {
- 'type': 'integer',
+ "http_proxy": {
+ "type": "object",
+ "properties": {
+ "port": {
+ "type": "integer",
},
- 'nfs_vhost_host': {
- 'type': 'string',
+ "nfs_vhost_host": {
+ "type": "string",
},
- 'nfs_vhost_port': {
- 'type': 'integer',
+ "nfs_vhost_port": {
+ "type": "integer",
},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
},
},
- 'additionalProperties': False,
+ "additionalProperties": False,
}
SQS_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'endpoint': {'type': 'string'},
- 'domain': {'type': 'string'},
- 'subdomain': {'type': 'string'},
- 'http_server': {
- 'type': 'object',
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "endpoint": {"type": "string"},
+ "domain": {"type": "string"},
+ "subdomain": {"type": "string"},
+ "http_server": {
+ "type": "object",
},
},
}
SOLOMON_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'domain': {'type': 'string'},
- 'subdomain': {'type': 'string'},
- 'volumes': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'name': {'type': 'string'},
- 'channels_profile': {'type': 'integer'},
- 'partitions': {'type': 'integer'},
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "domain": {"type": "string"},
+ "subdomain": {"type": "string"},
+ "volumes": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "channels_profile": {"type": "integer"},
+ "partitions": {"type": "integer"},
},
},
},
@@ -682,146 +682,146 @@ SOLOMON_SCHEMA = {
}
CMS_LIMITS_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'disabled_nodes_limit': {'type': 'integer'},
- 'disabled_nodes_ratio_limit': {'type': 'integer'},
+ "type": "object",
+ "properties": {
+ "disabled_nodes_limit": {"type": "integer"},
+ "disabled_nodes_ratio_limit": {"type": "integer"},
},
}
CMS_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'enable_sentinel': dict(type='boolean'),
- 'default_retry_time_sec': {'type': 'integer'},
- 'default_permission_duration_sec': {'type': 'integer'},
- 'info_collection_timeout_sec': {'type': 'integer'},
- 'tenant_limits': copy.deepcopy(CMS_LIMITS_SCHEMA),
- 'cluster_limits': copy.deepcopy(CMS_LIMITS_SCHEMA),
- 'monitors': {
- 'type': 'object',
- 'properties': {
- 'enable_auto_update': {'type': 'boolean'},
- 'update_interval_sec': {'type': 'integer'},
- 'ignored_downtime_gap_sec': {'type': 'integer'},
- 'broken_timeout_min': {'type': 'integer'},
- 'broken_prep_timeout_min': {'type': 'integer'},
- 'faulty_prep_timeout_min': {'type': 'integer'},
+ "type": "object",
+ "properties": {
+ "enable_sentinel": dict(type="boolean"),
+ "default_retry_time_sec": {"type": "integer"},
+ "default_permission_duration_sec": {"type": "integer"},
+ "info_collection_timeout_sec": {"type": "integer"},
+ "tenant_limits": copy.deepcopy(CMS_LIMITS_SCHEMA),
+ "cluster_limits": copy.deepcopy(CMS_LIMITS_SCHEMA),
+ "monitors": {
+ "type": "object",
+ "properties": {
+ "enable_auto_update": {"type": "boolean"},
+ "update_interval_sec": {"type": "integer"},
+ "ignored_downtime_gap_sec": {"type": "integer"},
+ "broken_timeout_min": {"type": "integer"},
+ "broken_prep_timeout_min": {"type": "integer"},
+ "faulty_prep_timeout_min": {"type": "integer"},
},
},
},
}
STATE_STORAGE = {
- 'type': 'object',
- 'properties': {
- 'allow_incorrect': {'type': 'boolean'},
- 'node_ids': {'type': 'array', 'items': {'type': 'integer'}},
- 'node_set': {'type': 'array', 'items': {'type': 'string'}},
+ "type": "object",
+ "properties": {
+ "allow_incorrect": {"type": "boolean"},
+ "node_ids": {"type": "array", "items": {"type": "integer"}},
+ "node_set": {"type": "array", "items": {"type": "string"}},
},
}
-METERING_SCHEMA = {'type': 'object', 'properties': {'metering_file': {'type': 'string'}}}
+METERING_SCHEMA = {"type": "object", "properties": {"metering_file": {"type": "string"}}}
YQL_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'endpoint': {'type': 'string'},
- 'database': {'type': 'string'},
- 'table_prefix': {'type': 'string'},
- 'oauth_file': {'type': 'string'},
- 'ydb_mvp_cloud_endpoint': {'type': 'string'},
- 'use_iam_from_metadata_service': {'type': 'boolean'},
- 'use_secure_connection': {'type': 'boolean'},
- 'use_bearer_for_ydb': {'type': 'boolean'},
- 'mdb_gateway': {'type': 'string'},
- 'mdb_transform_host': {'type': 'boolean'},
- 'object_storage_endpoint': {'type': 'string'},
- 'task_service_endpoint': {'type': 'string'},
- 'task_service_database': {'type': 'string'},
- 'impersonation_service_endpoint': {'type': 'string'},
- 'secure_task_service': {'type': 'boolean'},
- 'secure_impersonation_service': {'type': 'boolean'},
- 'hmac_secret_file': {'type': 'string'},
+ "type": "object",
+ "properties": {
+ "endpoint": {"type": "string"},
+ "database": {"type": "string"},
+ "table_prefix": {"type": "string"},
+ "oauth_file": {"type": "string"},
+ "ydb_mvp_cloud_endpoint": {"type": "string"},
+ "use_iam_from_metadata_service": {"type": "boolean"},
+ "use_secure_connection": {"type": "boolean"},
+ "use_bearer_for_ydb": {"type": "boolean"},
+ "mdb_gateway": {"type": "string"},
+ "mdb_transform_host": {"type": "boolean"},
+ "object_storage_endpoint": {"type": "string"},
+ "task_service_endpoint": {"type": "string"},
+ "task_service_database": {"type": "string"},
+ "impersonation_service_endpoint": {"type": "string"},
+ "secure_task_service": {"type": "boolean"},
+ "secure_impersonation_service": {"type": "boolean"},
+ "hmac_secret_file": {"type": "string"},
},
}
YQ_SCHEMA = {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'control_plane': {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'endpoint': {'type': 'string'},
- 'database': {'type': 'string'},
- 'table_prefix': {'type': 'string'},
- 'oauth_file': {'type': 'string'},
- 'certificate_file': {'type': 'string'},
- 'iam_endpoint': {'type': 'string'},
- 'sa_key_file': {'type': 'string'},
- 'use_local_metadata_service': {'type': 'boolean'},
- 'enable_forward_analytics': {'type': 'boolean'},
- 'enable_permissions': {'type': 'boolean'},
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "control_plane": {
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "endpoint": {"type": "string"},
+ "database": {"type": "string"},
+ "table_prefix": {"type": "string"},
+ "oauth_file": {"type": "string"},
+ "certificate_file": {"type": "string"},
+ "iam_endpoint": {"type": "string"},
+ "sa_key_file": {"type": "string"},
+ "use_local_metadata_service": {"type": "boolean"},
+ "enable_forward_analytics": {"type": "boolean"},
+ "enable_permissions": {"type": "boolean"},
},
},
- 'analytics': {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'endpoint': {'type': 'string'},
- 'database': {'type': 'string'},
- 'table_prefix': {'type': 'string'},
- 'oauth_file': {'type': 'string'},
- 'ydb_mvp_cloud_endpoint': {'type': 'string'},
- 'use_iam_from_metadata_service': {'type': 'boolean'},
- 'use_secure_connection': {'type': 'boolean'},
- 'use_bearer_for_ydb': {'type': 'boolean'},
- 'mdb_gateway': {'type': 'string'},
- 'mdb_transform_host': {'type': 'boolean'},
- 'object_storage_endpoint': {'type': 'string'},
- 'task_service_endpoint': {'type': 'string'},
- 'task_service_database': {'type': 'string'},
- 'impersonation_service_endpoint': {'type': 'string'},
- 'secure_task_service': {'type': 'boolean'},
- 'secure_impersonation_service': {'type': 'boolean'},
- 'hmac_secret_file': {'type': 'string'},
+ "analytics": {
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "endpoint": {"type": "string"},
+ "database": {"type": "string"},
+ "table_prefix": {"type": "string"},
+ "oauth_file": {"type": "string"},
+ "ydb_mvp_cloud_endpoint": {"type": "string"},
+ "use_iam_from_metadata_service": {"type": "boolean"},
+ "use_secure_connection": {"type": "boolean"},
+ "use_bearer_for_ydb": {"type": "boolean"},
+ "mdb_gateway": {"type": "string"},
+ "mdb_transform_host": {"type": "boolean"},
+ "object_storage_endpoint": {"type": "string"},
+ "task_service_endpoint": {"type": "string"},
+ "task_service_database": {"type": "string"},
+ "impersonation_service_endpoint": {"type": "string"},
+ "secure_task_service": {"type": "boolean"},
+ "secure_impersonation_service": {"type": "boolean"},
+ "hmac_secret_file": {"type": "string"},
},
},
- 'streaming': {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'endpoint': {'type': 'string'},
- 'database': {'type': 'string'},
- 'table_prefix': {'type': 'string'},
- 'oauth_file': {'type': 'string'},
- 'certificate_file': {'type': 'string'},
- 'hmac_secret_file': {'type': 'string'},
- 'iam_endpoint': {'type': 'string'},
- 'sa_key_file': {'type': 'string'},
- 'use_local_metadata_service': {'type': 'boolean'},
- 'checkpointing': {
- 'type': 'object',
- 'properties': {
- 'enabled': {'type': 'boolean'},
- 'period_millis': {'type': 'integer'},
- 'max_in_flight': {'type': 'integer'},
+ "streaming": {
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "endpoint": {"type": "string"},
+ "database": {"type": "string"},
+ "table_prefix": {"type": "string"},
+ "oauth_file": {"type": "string"},
+ "certificate_file": {"type": "string"},
+ "hmac_secret_file": {"type": "string"},
+ "iam_endpoint": {"type": "string"},
+ "sa_key_file": {"type": "string"},
+ "use_local_metadata_service": {"type": "boolean"},
+ "checkpointing": {
+ "type": "object",
+ "properties": {
+ "enabled": {"type": "boolean"},
+ "period_millis": {"type": "integer"},
+ "max_in_flight": {"type": "integer"},
},
},
},
},
- 'token_accessor': {
- 'type': 'object',
- 'properties': {'endpoint': {'type': 'string'}, 'use_ssl': {'type': 'boolean'}},
+ "token_accessor": {
+ "type": "object",
+ "properties": {"endpoint": {"type": "string"}, "use_ssl": {"type": "boolean"}},
},
- 'folder_service': {
- 'type': 'object',
- 'properties': {
- 'enable': {'type': 'boolean'},
- 'endpoint': {'type': 'string'},
- 'path_to_root_ca': {'type': 'string'},
+ "folder_service": {
+ "type": "object",
+ "properties": {
+ "enable": {"type": "boolean"},
+ "endpoint": {"type": "string"},
+ "path_to_root_ca": {"type": "string"},
},
},
},
@@ -830,54 +830,54 @@ YQ_SCHEMA = {
TEMPLATE_SCHEMA = {
"type": "object",
"properties": {
- 'enable_cores': dict(type='boolean'),
- 'state_storage': copy.deepcopy(STATE_STORAGE),
- "system_tablets": {'type': 'object'},
- "forbid_implicit_storage_pools": {'type': 'boolean'},
- "use_fixed_tablet_types": {'type': 'boolean'},
- "monitoring_address": {'type': 'string'},
- "use_console_feature": {'type': 'boolean'},
- 'storage_config_generation': {'type': 'integer'},
- 'use_walle': {'type': 'boolean'},
- 'cloud_mode': {'type': 'boolean'},
- 'host_configs': {
- 'type': 'array',
- 'items': {
- 'type': 'object',
- 'properties': {
- 'generation': {'type': 'integer', 'min': 0},
- 'drives': {
- 'type': 'array',
- 'items': copy.deepcopy(DRIVE_SCHEMA),
+ "enable_cores": dict(type="boolean"),
+ "state_storage": copy.deepcopy(STATE_STORAGE),
+ "system_tablets": {"type": "object"},
+ "forbid_implicit_storage_pools": {"type": "boolean"},
+ "use_fixed_tablet_types": {"type": "boolean"},
+ "monitoring_address": {"type": "string"},
+ "use_console_feature": {"type": "boolean"},
+ "storage_config_generation": {"type": "integer"},
+ "use_walle": {"type": "boolean"},
+ "cloud_mode": {"type": "boolean"},
+ "host_configs": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "generation": {"type": "integer", "min": 0},
+ "drives": {
+ "type": "array",
+ "items": copy.deepcopy(DRIVE_SCHEMA),
},
},
},
},
- 'static_bs_group_hosts': {
- 'type': 'array',
- 'items': {
- 'type': 'string',
+ "static_bs_group_hosts": {
+ "type": "array",
+ "items": {
+ "type": "string",
},
},
- 'use_cluster_uuid': {
- 'type': 'boolean',
+ "use_cluster_uuid": {
+ "type": "boolean",
},
- 'require_address': {
- 'type': 'boolean',
+ "require_address": {
+ "type": "boolean",
},
- 'cluster_uuid': {
- 'type': 'string',
- 'minLength': 1,
+ "cluster_uuid": {
+ "type": "string",
+ "minLength": 1,
},
- 'accepted_cluster_uuids': {'type': 'array', 'items': {'type': 'string', 'minLength': 1}},
- 'security_settings': {
- 'type': 'object',
- 'properties': {
- 'enforce_user_token_requirement': {
- 'type': 'boolean',
+ "accepted_cluster_uuids": {"type": "array", "items": {"type": "string", "minLength": 1}},
+ "security_settings": {
+ "type": "object",
+ "properties": {
+ "enforce_user_token_requirement": {
+ "type": "boolean",
},
- 'monitoring_allowed_sids': {'type': 'array', 'items': {'type': 'string'}},
- 'administration_allowed_sids': {'type': 'array', 'items': {'type': 'string'}},
+ "monitoring_allowed_sids": {"type": "array", "items": {"type": "string"}},
+ "administration_allowed_sids": {"type": "array", "items": {"type": "string"}},
},
},
"static_erasure": {
@@ -890,58 +890,58 @@ TEMPLATE_SCHEMA = {
"minLength": 1,
"enum": FailDomainType.all_fail_domain_type_names(),
},
- 'static_pdisk_type': {
- 'type': 'string',
- 'enum': PDiskCategory.all_pdisk_category_names(),
+ "static_pdisk_type": {
+ "type": "string",
+ "enum": PDiskCategory.all_pdisk_category_names(),
},
- 'nw_cache_file_path': {
- 'type': 'string',
+ "nw_cache_file_path": {
+ "type": "string",
},
- 'enable_cms_config_cache': {
- 'type': 'boolean',
+ "enable_cms_config_cache": {
+ "type": "boolean",
},
"hosts": {
"type": "array",
"items": copy.deepcopy(HOST_SCHEMA),
- 'minItems': 1,
- 'checkNameServiceDuplicates': True,
- },
- 'auth': {'type': 'object'},
- 'log': copy.deepcopy(LOG_SCHEMA),
- 'grpc': {'type': 'object'},
- 'kqp': copy.deepcopy(KQP_SCHEMA),
- 'ic': {'type': 'object'},
- 'pq': {'type': 'object'},
- 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL)},
- 'profiles': copy.deepcopy(PROFILES),
- 'domains': {'type': 'array', 'items': copy.deepcopy(DOMAIN_SCHEMA), 'minItems': 1},
- 'nbs': copy.deepcopy(NBS_SCHEMA),
- 'nbs_control': copy.deepcopy(NBS_SCHEMA),
- 'nfs': copy.deepcopy(NFS_SCHEMA),
- 'nfs_control': copy.deepcopy(NFS_SCHEMA),
- 'sqs': copy.deepcopy(SQS_SCHEMA),
- 'features': copy.deepcopy(FEATURES_SCHEMA),
- 'shared_cache': copy.deepcopy(SHARED_CACHE_SCHEMA),
- 'sys': copy.deepcopy(SYS_SCHEMA),
- 'tracing': copy.deepcopy(TRACING_SCHEMA),
- 'failure_injection_config': copy.deepcopy(FAILURE_INJECTION_CONFIG_SCHEMA),
- 'solomon': copy.deepcopy(SOLOMON_SCHEMA),
- 'cms': copy.deepcopy(CMS_SCHEMA),
- 'resource_broker': {'type': 'object'},
- 'state_storages': {
- 'type': 'array',
- 'items': {'type': 'object', 'properties': {}, 'additionalProperties': True},
- },
- 'metering': copy.deepcopy(METERING_SCHEMA),
- 'yql_analytics': copy.deepcopy(YQL_SCHEMA),
- 'yq': copy.deepcopy(YQ_SCHEMA),
+ "minItems": 1,
+ "checkNameServiceDuplicates": True,
+ },
+ "auth": {"type": "object"},
+ "log": copy.deepcopy(LOG_SCHEMA),
+ "grpc": {"type": "object"},
+ "kqp": copy.deepcopy(KQP_SCHEMA),
+ "ic": {"type": "object"},
+ "pq": {"type": "object"},
+ "storage_pools": {"type": "array", "items": copy.deepcopy(STORAGE_POOL)},
+ "profiles": copy.deepcopy(PROFILES),
+ "domains": {"type": "array", "items": copy.deepcopy(DOMAIN_SCHEMA), "minItems": 1},
+ "nbs": copy.deepcopy(NBS_SCHEMA),
+ "nbs_control": copy.deepcopy(NBS_SCHEMA),
+ "nfs": copy.deepcopy(NFS_SCHEMA),
+ "nfs_control": copy.deepcopy(NFS_SCHEMA),
+ "sqs": copy.deepcopy(SQS_SCHEMA),
+ "features": copy.deepcopy(FEATURES_SCHEMA),
+ "shared_cache": copy.deepcopy(SHARED_CACHE_SCHEMA),
+ "sys": copy.deepcopy(SYS_SCHEMA),
+ "tracing": copy.deepcopy(TRACING_SCHEMA),
+ "failure_injection_config": copy.deepcopy(FAILURE_INJECTION_CONFIG_SCHEMA),
+ "solomon": copy.deepcopy(SOLOMON_SCHEMA),
+ "cms": copy.deepcopy(CMS_SCHEMA),
+ "resource_broker": {"type": "object"},
+ "state_storages": {
+ "type": "array",
+ "items": {"type": "object", "properties": {}, "additionalProperties": True},
+ },
+ "metering": copy.deepcopy(METERING_SCHEMA),
+ "yql_analytics": copy.deepcopy(YQL_SCHEMA),
+ "yq": copy.deepcopy(YQ_SCHEMA),
},
"required": ["static_erasure", "hosts"],
}
def _host_and_ic_port(host):
- return "%s:%s" % (host['name'], str(host.get('ic_port', 19001)))
+ return "%s:%s" % (host["name"], str(host.get("ic_port", 19001)))
def checkNameServiceDuplicates(validator, allow_duplicates, instance, schema):
@@ -962,7 +962,7 @@ _Validator = jsonschema.Draft4Validator
_Validator = jsonschema.validators.extend(
_Validator,
{
- 'checkNameServiceDuplicates': checkNameServiceDuplicates,
+ "checkNameServiceDuplicates": checkNameServiceDuplicates,
},
)
@@ -973,6 +973,9 @@ class Validator(_Validator):
super(Validator, self).__init__(schema, format_checker=format_checker)
+def default_validator():
+ return Validator(copy.deepcopy(TEMPLATE_SCHEMA))
+
+
def validate(template):
- schema = copy.deepcopy(TEMPLATE_SCHEMA)
- Validator(schema).validate(template)
+ default_validator().validate(template)
diff --git a/ydb/tools/cfg/walle/__init__.py b/ydb/tools/cfg/walle/__init__.py
index df7d52021c..808bc9e2bd 100644
--- a/ydb/tools/cfg/walle/__init__.py
+++ b/ydb/tools/cfg/walle/__init__.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-from .walle import HostsFetcherInterface, NopHostsFetcher # noqa
+from .walle import HostsInformationProvider, NopHostsInformationProvider, WalleHostsInformationProvider # noqa
-__all__ = ('Walle',)
+__all__ = ("Walle",)
diff --git a/ydb/tools/cfg/walle/walle.py b/ydb/tools/cfg/walle/walle.py
index 92792aaf6e..e2677f48a9 100644
--- a/ydb/tools/cfg/walle/walle.py
+++ b/ydb/tools/cfg/walle/walle.py
@@ -1,10 +1,19 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+import os
+import requests
+import threading
import zlib
+import json
+import time
+
+
from abc import ABCMeta, abstractmethod
+from six.moves.urllib import parse
+
-class HostsFetcherInterface:
+class HostsInformationProvider:
__metaclass__ = ABCMeta
@abstractmethod
@@ -20,7 +29,7 @@ class HostsFetcherInterface:
pass
-class NopHostsFetcher(HostsFetcherInterface):
+class NopHostsInformationProvider(HostsInformationProvider):
def __init__(self):
pass
@@ -32,3 +41,65 @@ class NopHostsFetcher(HostsFetcherInterface):
def get_body(self, hostname):
return zlib.crc32(hostname.encode())
+
+
+class WalleHostsInformationProvider(HostsInformationProvider):
+ def __init__(self, provider_url=None, cloud_mode=False):
+ if provider_url is None:
+ raise RuntimeError("Got empty hosts_provider url")
+ self._base_url = provider_url
+ self._cache = {}
+ self._timeout_seconds = 5
+ self._retry_count = 10
+ self._cloud_mode = cloud_mode
+ self._lock = threading.Lock()
+
+ def _ask_location(self, hostname):
+ with self._lock:
+ if hostname in self._cache:
+ return self._cache[hostname]
+
+ with requests.Session() as session:
+ retries = 10
+ url = self._base_url + hostname + "?" + parse.urlencode({"fields": "location,name,inv,short_queue_name"})
+ headers = dict()
+ if "YC_TOKEN" in os.environ:
+ headers["Authorization"] = "Bearer " + os.environ.get("YC_TOKEN").strip()
+
+ while retries > 0:
+ retries -= 1
+
+ try:
+ response = session.get(url, timeout=self._timeout_seconds, headers=headers)
+ response.raise_for_status()
+ loaded = json.loads(response.content)
+ with self._lock:
+ self._cache[hostname] = loaded
+ return self._cache[hostname]
+
+ except IOError:
+ if retries == 0:
+ raise RuntimeError(
+ "Failed to retrieve information about host %s in %d retries"
+ % (
+ hostname,
+ self._retry_count,
+ )
+ )
+
+ time.sleep(3)
+ raise RuntimeError()
+
+ def get_rack(self, hostname):
+ short_queue_name = self._ask_location(hostname)["location"]["short_queue_name"]
+ rack = self._ask_location(hostname)["location"]["rack"]
+ if self._cloud_mode:
+ return "{}:{}".format(short_queue_name.lower(), rack.lower())
+ return "{}#{}".format(short_queue_name.upper(), rack)
+
+ def get_datacenter(self, hostname):
+ short_dc_name = self._ask_location(hostname)["location"]["short_datacenter_name"]
+ return short_dc_name if self._cloud_mode else short_dc_name.upper()
+
+ def get_body(self, hostname):
+ return self._ask_location(hostname)["inv"]