diff options
author | shmel1k <shmel1k@ydb.tech> | 2023-11-28 17:59:19 +0300 |
---|---|---|
committer | shmel1k <shmel1k@ydb.tech> | 2023-11-28 19:21:01 +0300 |
commit | fc839da62d0e112f26540ebebaf964d93890f2fb (patch) | |
tree | 31d751464c357aa97cfbea4ea70cf315480b9ab7 | |
parent | 10ae15c98d33118e47701756828019e97aa10ec6 (diff) | |
download | ydb-fc839da62d0e112f26540ebebaf964d93890f2fb.tar.gz |
add Walle opensource adapter
-rw-r--r-- | ydb/tools/cfg/base.py | 2 | ||||
-rw-r--r-- | ydb/tools/cfg/bin/__main__.py | 35 | ||||
-rw-r--r-- | ydb/tools/cfg/validation.py | 1213 | ||||
-rw-r--r-- | ydb/tools/cfg/walle/__init__.py | 4 | ||||
-rw-r--r-- | ydb/tools/cfg/walle/walle.py | 75 |
5 files changed, 704 insertions, 625 deletions
diff --git a/ydb/tools/cfg/base.py b/ydb/tools/cfg/base.py index 3bf9c8a0a8..63d1cbdf97 100644 --- a/ydb/tools/cfg/base.py +++ b/ydb/tools/cfg/base.py @@ -266,7 +266,7 @@ def normalize_domain(domain_name): class ClusterDetailsProvider(object): def __init__(self, template, walle_provider, validator=None, database=None): if not validator: - validator = validation.Validator() + validator = validation.default_validator() self.__validator = validator diff --git a/ydb/tools/cfg/bin/__main__.py b/ydb/tools/cfg/bin/__main__.py index a4c6156311..c76379907f 100644 --- a/ydb/tools/cfg/bin/__main__.py +++ b/ydb/tools/cfg/bin/__main__.py @@ -11,24 +11,25 @@ from ydb.tools.cfg.configurator_setup import get_parser, parse_optional_argument from ydb.tools.cfg.dynamic import DynamicConfigGenerator from ydb.tools.cfg.static import StaticConfigGenerator from ydb.tools.cfg.utils import write_to_file +from ydb.tools.cfg.walle import NopHostsInformationProvider, WalleHostsInformationProvider logging_config.dictConfig( { - 'version': 1, - 'formatters': { - 'base': { - 'format': '%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s', + "version": 1, + "formatters": { + "base": { + "format": "%(asctime)s - %(processName)s - %(name)s - %(levelname)s - %(message)s", }, }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - 'formatter': 'base', - 'stream': sys.stdout, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "base", + "stream": sys.stdout, }, }, - 'root': {'formatter': 'base', 'level': 'INFO', 'handlers': ('console',)}, + "root": {"formatter": "base", "level": "INFO", "handlers": ("console",)}, } ) @@ -44,10 +45,14 @@ def cfg_generate(args): else: cfg_cls = StaticConfigGenerator - with open(args.cluster_description, 'r') as yaml_template: + with open(args.cluster_description, "r") as yaml_template: cluster_template = yaml.safe_load(yaml_template) - generator = cfg_cls(cluster_template, args.binary_path, args.output_dir, **kwargs) + hosts_provider = NopHostsInformationProvider() + if args.hosts_provider_url: + hosts_provider = WalleHostsInformationProvider(args.hosts_provider_url) + + generator = cfg_cls(cluster_template, args.binary_path, args.output_dir, walle_provider=hosts_provider, **kwargs) all_configs = generator.get_all_configs() for cfg_name, cfg_value in all_configs.items(): @@ -55,10 +60,10 @@ def cfg_generate(args): def main(): - parser = get_parser(cfg_generate) + parser = get_parser(cfg_generate, [{"name": "--hosts-provider-url", "help": "URL from which information about hosts can be obtained."}]) args = parser.parse_args() args.func(args) -if __name__ == '__main__': +if __name__ == "__main__": exit(main()) diff --git a/ydb/tools/cfg/validation.py b/ydb/tools/cfg/validation.py index c764e2b372..8923291264 100644 --- a/ydb/tools/cfg/validation.py +++ b/ydb/tools/cfg/validation.py @@ -8,9 +8,9 @@ from ydb.tools.cfg import utils from ydb.tools.cfg.types import Erasure, FailDomainType, LogLevels, NodeType, PDiskCategory KQP_SCHEMA = { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, + "type": "object", + "properties": { + "enable": {"type": "boolean"}, }, } @@ -18,28 +18,28 @@ KQP_SCHEMA = { # These profiles will be on top of channels.txt file so # user can influence on id of these profiles PROFILES = { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'channels': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'pdisk_type': { - 'type': 'string', - 'enum': PDiskCategory.all_pdisk_category_names(), + "type": "array", + "items": { + "type": "object", + "properties": { + "channels": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pdisk_type": { + "type": "string", + "enum": PDiskCategory.all_pdisk_category_names(), }, - 'erasure': { - 'type': 'string', - 'enum': Erasure.all_erasure_type_names(), + "erasure": { + "type": "string", + "enum": Erasure.all_erasure_type_names(), }, - 'vdisk_kind': { - 'type': 'string', + "vdisk_kind": { + "type": "string", }, - 'storage_pool_kind': { - 'type': 'string', + "storage_pool_kind": { + "type": "string", }, }, "additionalProperties": False, @@ -52,111 +52,111 @@ PROFILES = { FEATURES_SCHEMA = { - 'type': 'object', - 'properties': {}, + "type": "object", + "properties": {}, } EXECUTOR_SCHEMA = { - 'type': 'object', - 'properties': { - 'threads': { - 'type': "integer", - 'min': 1, - }, - 'spin_threshold': { - "type": 'integer', + "type": "object", + "properties": { + "threads": { + "type": "integer", "min": 1, }, - 'TimePerMailboxMicroSecs': { - 'type': 'integer', + "spin_threshold": { + "type": "integer", "min": 1, }, - 'time_per_mailbox_micro_secs': { - 'type': 'integer', + "TimePerMailboxMicroSecs": { + "type": "integer", + "min": 1, + }, + "time_per_mailbox_micro_secs": { + "type": "integer", "min": 1, }, - 'max_threads': { - 'type': 'integer', - 'min': 1, + "max_threads": { + "type": "integer", + "min": 1, }, - 'priority': { - 'type': 'integer', - 'min': 1, + "priority": { + "type": "integer", + "min": 1, }, - 'max_avg_ping_deviation': { - 'type': 'integer', - 'min': 1, + "max_avg_ping_deviation": { + "type": "integer", + "min": 1, }, }, } SYS_SCHEMA = { - 'type': 'object', - 'properties': { - 'preset_name': { - 'type': 'string', - 'enum': utils.get_resources_list('resources/sys/'), - }, - 'executors': { - 'type': 'object', - 'properties': { - 'system': copy.deepcopy(EXECUTOR_SCHEMA), - 'batch': copy.deepcopy(EXECUTOR_SCHEMA), - 'user': copy.deepcopy(EXECUTOR_SCHEMA), - 'io': copy.deepcopy(EXECUTOR_SCHEMA), - 'ic': copy.deepcopy(EXECUTOR_SCHEMA), + "type": "object", + "properties": { + "preset_name": { + "type": "string", + "enum": utils.get_resources_list("resources/sys/"), + }, + "executors": { + "type": "object", + "properties": { + "system": copy.deepcopy(EXECUTOR_SCHEMA), + "batch": copy.deepcopy(EXECUTOR_SCHEMA), + "user": copy.deepcopy(EXECUTOR_SCHEMA), + "io": copy.deepcopy(EXECUTOR_SCHEMA), + "ic": copy.deepcopy(EXECUTOR_SCHEMA), }, - 'additionalProperties': False, - }, - 'scheduler': { - 'type': 'object', - 'properties': { - 'resolution': {'type': 'integer'}, - 'spin_threshold': {'type': 'integer'}, - 'progress_threshold': {'type': 'integer'}, + "additionalProperties": False, + }, + "scheduler": { + "type": "object", + "properties": { + "resolution": {"type": "integer"}, + "spin_threshold": {"type": "integer"}, + "progress_threshold": {"type": "integer"}, }, }, - 'use_auto_config': {'type': 'boolean'}, - 'cpu_count': {'type': 'integer'}, - 'node_type': { - 'type': 'string', - 'enum': NodeType.all_node_type_names(), + "use_auto_config": {"type": "boolean"}, + "cpu_count": {"type": "integer"}, + "node_type": { + "type": "string", + "enum": NodeType.all_node_type_names(), }, }, - 'additionalProperties': False, + "additionalProperties": False, } TRACING_SCHEMA = dict( - type='object', + type="object", properties=dict( - host=dict(type='string'), - port=dict(type='integer'), - root_ca=dict(type='string'), - service_name=dict(type='string'), + host=dict(type="string"), + port=dict(type="integer"), + root_ca=dict(type="string"), + service_name=dict(type="string"), ), required=[ - 'host', - 'port', - 'root_ca', - 'service_name', + "host", + "port", + "root_ca", + "service_name", ], additionalProperties=False, ) FAILURE_INJECTION_CONFIG_SCHEMA = { "type": "object", - "properties": {'approximate_termination_interval': dict(type='integer')}, + "properties": {"approximate_termination_interval": dict(type="integer")}, "additionalProperties": False, } DRIVE_SCHEMA = { "type": "object", "properties": { - 'type': dict(type='string', enum=PDiskCategory.all_pdisk_category_names()), - 'path': dict(type='string', minLength=1), - 'shared_with_os': dict(type='boolean'), - 'expected_slot_count': dict(type='integer'), - 'kind': dict(type='integer'), + "type": dict(type="string", enum=PDiskCategory.all_pdisk_category_names()), + "path": dict(type="string", minLength=1), + "shared_with_os": dict(type="boolean"), + "expected_slot_count": dict(type="integer"), + "kind": dict(type="integer"), }, "required": ["type", "path"], "additionalProperties": False, @@ -167,7 +167,7 @@ HOST_SCHEMA = { "properties": { "name": {"type": "string", "minLength": 1}, "drives": {"type": "array", "items": copy.deepcopy(DRIVE_SCHEMA)}, - 'host_config_id': {'type': "integer", 'minLength': 1}, + "host_config_id": {"type": "integer", "minLength": 1}, "ic_port": { "type": "integer", }, @@ -178,7 +178,7 @@ HOST_SCHEMA = { } LOG_SCHEMA = { - 'type': "object", + "type": "object", "properties": { "default": {"type": "integer", "min": int(min(LogLevels)), "max": int(max(LogLevels))}, "entries": { @@ -199,59 +199,59 @@ LOG_SCHEMA = { } STORAGE_POOL = { - 'type': 'object', - 'properties': { - 'erasure': { - 'type': 'string', - 'minLength': 1, - 'enum': Erasure.all_erasure_type_names(), - }, - 'generation': {'type': 'integer', 'min': 0}, - 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1}, - 'num_groups': { - 'type': 'integer', - 'min': 1, - }, - 'storage_pool_id': {'type': 'integer', 'min': 1}, + "type": "object", + "properties": { + "erasure": { + "type": "string", + "minLength": 1, + "enum": Erasure.all_erasure_type_names(), + }, + "generation": {"type": "integer", "min": 0}, + "encryption_mode": {"type": "integer", "min": 0, "max": 1}, + "num_groups": { + "type": "integer", + "min": 1, + }, + "storage_pool_id": {"type": "integer", "min": 1}, "fail_domain_type": { "type": "string", "minLength": 1, "enum": FailDomainType.all_fail_domain_type_names(), }, - 'kind': { - 'type': 'string', - 'minLength': 1, + "kind": { + "type": "string", + "minLength": 1, }, - 'name': { - 'type': 'string', - 'minLength': 1, + "name": { + "type": "string", + "minLength": 1, }, - 'filter_properties': { - 'type': 'object', - 'properties': { - 'type': { - 'type': 'string', - 'enum': PDiskCategory.all_pdisk_category_names(), + "filter_properties": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": PDiskCategory.all_pdisk_category_names(), }, - 'SharedWithOs': { - 'type': 'boolean', + "SharedWithOs": { + "type": "boolean", }, - 'kind': dict(type='integer'), + "kind": dict(type="integer"), }, - 'required': ['type'], - 'additionalProperties': False, + "required": ["type"], + "additionalProperties": False, }, - 'vdisk_kind': { - 'type': 'string', - 'minLength': 1, + "vdisk_kind": { + "type": "string", + "minLength": 1, }, }, - 'required': [ - 'num_groups', - 'erasure', - 'filter_properties', + "required": [ + "num_groups", + "erasure", + "filter_properties", ], - 'additionalProperties': False, + "additionalProperties": False, } STORAGE_POOL_KIND = { @@ -259,14 +259,14 @@ STORAGE_POOL_KIND = { "properties": { "kind": { "type": "string", - 'minLength': 1, + "minLength": 1, }, "erasure": { "type": "string", "minLength": 1, "enum": Erasure.all_erasure_type_names(), }, - 'encryption_mode': {'type': 'integer', 'min': 0, 'max': 1}, + "encryption_mode": {"type": "integer", "min": 0, "max": 1}, "filter_properties": { "type": "object", "properties": { @@ -274,10 +274,10 @@ STORAGE_POOL_KIND = { "type": "string", "enum": PDiskCategory.all_pdisk_category_names(), }, - 'SharedWithOs': { - 'type': 'boolean', + "SharedWithOs": { + "type": "boolean", }, - 'kind': dict(type='integer'), + "kind": dict(type="integer"), }, "required": ["type"], "additionalProperties": False, @@ -287,299 +287,299 @@ STORAGE_POOL_KIND = { "minLength": 1, "enum": FailDomainType.all_fail_domain_type_names(), }, - 'vdisk_kind': { - 'type': 'string', - 'minLength': 1, + "vdisk_kind": { + "type": "string", + "minLength": 1, }, }, "required": ["kind", "erasure", "filter_properties"], } STORAGE_POOL_INSTANCE = { - 'type': 'object', - 'properties': { - 'kind': { - 'type': 'string', - 'minLength': 1, + "type": "object", + "properties": { + "kind": { + "type": "string", + "minLength": 1, }, - 'num_groups': { - 'type': 'integer', - 'min': 1, + "num_groups": { + "type": "integer", + "min": 1, }, - 'generation': { - 'type': 'integer', - 'min': 0, + "generation": { + "type": "integer", + "min": 0, }, }, - 'required': [ - 'kind', - 'num_groups', + "required": [ + "kind", + "num_groups", ], - 'additionalProperties': False, + "additionalProperties": False, } SHARED_CACHE_SCHEMA = { - 'type': 'object', - 'properties': {'memory_limit': {'type': 'integer'}}, - 'additionalProperties': False, + "type": "object", + "properties": {"memory_limit": {"type": "integer"}}, + "additionalProperties": False, } TENANT_SCHEMA = { - 'type': 'object', - 'properties': { - 'name': { - 'type': 'string', - 'minLength': 1, - }, - 'storage_units': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'count': { - 'type': 'integer', + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + }, + "storage_units": { + "type": "array", + "items": { + "type": "object", + "properties": { + "count": { + "type": "integer", }, - 'kind': {'type': 'string', 'minLength': 1}, + "kind": {"type": "string", "minLength": 1}, }, }, }, - 'compute_units': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'count': { - 'type': 'integer', - 'min': 1, + "compute_units": { + "type": "array", + "items": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "min": 1, }, - 'kind': { - 'type': 'string', - 'minLength': 1, + "kind": { + "type": "string", + "minLength": 1, }, - 'zone': {'type': 'string', 'minLength': 1}, - 'required': ['count', 'kind', 'zone'], - 'additionalProperties': False, + "zone": {"type": "string", "minLength": 1}, + "required": ["count", "kind", "zone"], + "additionalProperties": False, }, }, }, - 'overridden_configs': { - 'type': 'object', + "overridden_configs": { + "type": "object", }, }, } DOMAIN_SCHEMA = { - 'type': 'object', - 'properties': { - 'domain_name': {'type': 'string', 'minLength': 1}, - 'plan_resolution': { - 'type': 'integer', - 'min': 1, - }, - 'domain_id': {'type': 'integer', 'min': 1}, - 'mediators': { - 'type': 'integer', - 'min': 1, - }, - 'coordinators': {'type': 'integer', 'min': 1}, - 'allocators': {'type': 'integer', 'min': 1}, - 'storage_pool_kinds': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_KIND)}, - 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL_INSTANCE)}, - 'databases': { - 'type': 'array', - 'items': copy.deepcopy(TENANT_SCHEMA), - }, - 'config_cookie': { - 'type': 'string', - }, - 'dynamic_slots': {'type': 'integer', 'min': 1}, - 'bind_slots_to_numa_nodes': {'type': 'boolean'}, - 'console_initializers': { - 'type': 'array', - 'items': { - 'type': 'string', - 'enum': utils.get_resources_list('resources/console_initializers/'), + "type": "object", + "properties": { + "domain_name": {"type": "string", "minLength": 1}, + "plan_resolution": { + "type": "integer", + "min": 1, + }, + "domain_id": {"type": "integer", "min": 1}, + "mediators": { + "type": "integer", + "min": 1, + }, + "coordinators": {"type": "integer", "min": 1}, + "allocators": {"type": "integer", "min": 1}, + "storage_pool_kinds": {"type": "array", "items": copy.deepcopy(STORAGE_POOL_KIND)}, + "storage_pools": {"type": "array", "items": copy.deepcopy(STORAGE_POOL_INSTANCE)}, + "databases": { + "type": "array", + "items": copy.deepcopy(TENANT_SCHEMA), + }, + "config_cookie": { + "type": "string", + }, + "dynamic_slots": {"type": "integer", "min": 1}, + "bind_slots_to_numa_nodes": {"type": "boolean"}, + "console_initializers": { + "type": "array", + "items": { + "type": "string", + "enum": utils.get_resources_list("resources/console_initializers/"), }, }, }, - 'required': ['domain_name'], - 'additionalProperties': False, + "required": ["domain_name"], + "additionalProperties": False, } NBS_SCHEMA = { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'new_config_generator_enabled': {'type': 'boolean'}, - 'sys': copy.deepcopy(SYS_SCHEMA), - 'log': copy.deepcopy(LOG_SCHEMA), - 'domain': {'type': 'string'}, - 'subdomain': {'type': 'string'}, - 'storage': {'type': 'object', 'properties': {}}, - 'disk_registry_proxy': { - 'type': 'object', - }, - 'server': { - 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', - }, - 'port': { - 'type': 'integer', - }, - 'root_certs_file': { - 'type': 'string', - }, - 'certs': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'cert_file': { - 'type': 'string', + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "new_config_generator_enabled": {"type": "boolean"}, + "sys": copy.deepcopy(SYS_SCHEMA), + "log": copy.deepcopy(LOG_SCHEMA), + "domain": {"type": "string"}, + "subdomain": {"type": "string"}, + "storage": {"type": "object", "properties": {}}, + "disk_registry_proxy": { + "type": "object", + }, + "server": { + "type": "object", + "properties": { + "host": { + "type": "string", + }, + "port": { + "type": "integer", + }, + "root_certs_file": { + "type": "string", + }, + "certs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "cert_file": { + "type": "string", }, - 'cert_private_key_file': { - 'type': 'string', + "cert_private_key_file": { + "type": "string", }, }, }, }, }, }, - 'discovery': {'type': 'object'}, - 'ydbstats': {'type': 'object'}, - 'auth': {'type': 'object'}, - 'names': {'type': 'object'}, - 'client': { - 'type': 'object', - 'properties': { - 'auth_config': {'type': 'object'}, - 'client_config': {'type': 'object'}, - 'log_config': {'type': 'object'}, - 'monitoring_config': {'type': 'object'}, - 'throttling_enabled': {'type': 'boolean'}, - 'throttling_enabled_s_s_d': {'type': 'boolean'}, - 'throttling_disabled_s_s_d_nonrepl': {'type': 'boolean'}, + "discovery": {"type": "object"}, + "ydbstats": {"type": "object"}, + "auth": {"type": "object"}, + "names": {"type": "object"}, + "client": { + "type": "object", + "properties": { + "auth_config": {"type": "object"}, + "client_config": {"type": "object"}, + "log_config": {"type": "object"}, + "monitoring_config": {"type": "object"}, + "throttling_enabled": {"type": "boolean"}, + "throttling_enabled_s_s_d": {"type": "boolean"}, + "throttling_disabled_s_s_d_nonrepl": {"type": "boolean"}, }, - 'additionalProperties': False, - }, - 'http_proxy': { - 'type': 'object', - 'properties': { - 'port': { - 'type': 'integer', - }, - 'secure_port': { - 'type': 'integer', - }, - 'certs': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'cert_file': { - 'type': 'string', + "additionalProperties": False, + }, + "http_proxy": { + "type": "object", + "properties": { + "port": { + "type": "integer", + }, + "secure_port": { + "type": "integer", + }, + "certs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "cert_file": { + "type": "string", }, - 'cert_private_key_file': { - 'type': 'string', + "cert_private_key_file": { + "type": "string", }, }, }, }, - 'nbs_server_host': { - 'type': 'string', + "nbs_server_host": { + "type": "string", }, - 'nbs_server_port': { - 'type': 'integer', + "nbs_server_port": { + "type": "integer", }, - 'nbs_server_cert_file': { - 'type': 'string', + "nbs_server_cert_file": { + "type": "string", }, - 'root_certs_file': { - 'type': 'string', + "root_certs_file": { + "type": "string", }, - 'nbs_server_insecure': { - 'type': 'boolean', + "nbs_server_insecure": { + "type": "boolean", }, }, - 'additionalProperties': False, + "additionalProperties": False, }, - 'breakpad': { - 'type': 'object', - 'properties': { - 'enable': { - 'type': 'boolean', + "breakpad": { + "type": "object", + "properties": { + "enable": { + "type": "boolean", }, }, - 'additionalProperties': False, + "additionalProperties": False, }, - 'breakpad_sender': { - 'type': 'object', - 'properties': { - 'aggregator_url': { - 'type': 'string', + "breakpad_sender": { + "type": "object", + "properties": { + "aggregator_url": { + "type": "string", }, - 'notify_email': { - 'type': 'string', + "notify_email": { + "type": "string", }, }, - 'additionalProperties': False, - }, - 'logbroker': { - 'type': 'object', - 'properties': { - 'address': {'type': 'string'}, - 'port': {'type': 'integer'}, - 'database': {'type': 'string'}, - 'use_logbroker_c_d_s': {'type': 'boolean'}, - 'ca_cert_filename': {'type': 'string'}, - 'topic': {'type': 'string'}, - 'source_id': {'type': 'string'}, - 'metadata_server_address': {'type': 'string'}, + "additionalProperties": False, + }, + "logbroker": { + "type": "object", + "properties": { + "address": {"type": "string"}, + "port": {"type": "integer"}, + "database": {"type": "string"}, + "use_logbroker_c_d_s": {"type": "boolean"}, + "ca_cert_filename": {"type": "string"}, + "topic": {"type": "string"}, + "source_id": {"type": "string"}, + "metadata_server_address": {"type": "string"}, }, - 'additionalProperties': False, + "additionalProperties": False, }, - 'notify': {'type': 'object', 'properties': {'endpoint': {'type': 'string'}}}, + "notify": {"type": "object", "properties": {"endpoint": {"type": "string"}}}, "iam": { - 'type': 'object', + "type": "object", }, "kms": { - 'type': 'object', + "type": "object", }, "compute": { - 'type': 'object', - }, - 'features': { - 'type': 'object', - 'properties': { - 'features': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'name': { - 'type': 'string', + "type": "object", + }, + "features": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", }, - 'Whitelist': { - 'type': 'object', - 'properties': { - 'cloud_ids': { - 'type': 'array', - 'items': { - 'type': 'string', + "Whitelist": { + "type": "object", + "properties": { + "cloud_ids": { + "type": "array", + "items": { + "type": "string", }, } }, }, - 'Blacklist': { - 'type': 'object', - 'properties': { - 'cloud_ids': { - 'type': 'array', - 'items': { - 'type': 'string', + "Blacklist": { + "type": "object", + "properties": { + "cloud_ids": { + "type": "array", + "items": { + "type": "string", }, } }, @@ -588,93 +588,93 @@ NBS_SCHEMA = { }, }, }, - 'additionalProperties': False, + "additionalProperties": False, }, }, - 'additionalProperties': False, + "additionalProperties": False, } NFS_SCHEMA = { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'new_config_generator_enabled': {'type': 'boolean'}, - 'sys': copy.deepcopy(SYS_SCHEMA), - 'log': copy.deepcopy(LOG_SCHEMA), - 'domain': {'type': 'string'}, - 'subdomain': {'type': 'string'}, - 'names': {'type': 'object'}, - 'storage': {'type': 'object'}, - 'diagnostics': {'type': 'object'}, - 'auth': {'type': 'object'}, - 'server': { - 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', - }, - 'port': { - 'type': 'integer', + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "new_config_generator_enabled": {"type": "boolean"}, + "sys": copy.deepcopy(SYS_SCHEMA), + "log": copy.deepcopy(LOG_SCHEMA), + "domain": {"type": "string"}, + "subdomain": {"type": "string"}, + "names": {"type": "object"}, + "storage": {"type": "object"}, + "diagnostics": {"type": "object"}, + "auth": {"type": "object"}, + "server": { + "type": "object", + "properties": { + "host": { + "type": "string", + }, + "port": { + "type": "integer", }, }, }, - 'vhost': { - 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', + "vhost": { + "type": "object", + "properties": { + "host": { + "type": "string", }, - 'port': { - 'type': 'integer', + "port": { + "type": "integer", }, }, }, - 'http_proxy': { - 'type': 'object', - 'properties': { - 'port': { - 'type': 'integer', + "http_proxy": { + "type": "object", + "properties": { + "port": { + "type": "integer", }, - 'nfs_vhost_host': { - 'type': 'string', + "nfs_vhost_host": { + "type": "string", }, - 'nfs_vhost_port': { - 'type': 'integer', + "nfs_vhost_port": { + "type": "integer", }, }, - 'additionalProperties': False, + "additionalProperties": False, }, }, - 'additionalProperties': False, + "additionalProperties": False, } SQS_SCHEMA = { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'endpoint': {'type': 'string'}, - 'domain': {'type': 'string'}, - 'subdomain': {'type': 'string'}, - 'http_server': { - 'type': 'object', + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "endpoint": {"type": "string"}, + "domain": {"type": "string"}, + "subdomain": {"type": "string"}, + "http_server": { + "type": "object", }, }, } SOLOMON_SCHEMA = { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'domain': {'type': 'string'}, - 'subdomain': {'type': 'string'}, - 'volumes': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'channels_profile': {'type': 'integer'}, - 'partitions': {'type': 'integer'}, + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "domain": {"type": "string"}, + "subdomain": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "channels_profile": {"type": "integer"}, + "partitions": {"type": "integer"}, }, }, }, @@ -682,146 +682,146 @@ SOLOMON_SCHEMA = { } CMS_LIMITS_SCHEMA = { - 'type': 'object', - 'properties': { - 'disabled_nodes_limit': {'type': 'integer'}, - 'disabled_nodes_ratio_limit': {'type': 'integer'}, + "type": "object", + "properties": { + "disabled_nodes_limit": {"type": "integer"}, + "disabled_nodes_ratio_limit": {"type": "integer"}, }, } CMS_SCHEMA = { - 'type': 'object', - 'properties': { - 'enable_sentinel': dict(type='boolean'), - 'default_retry_time_sec': {'type': 'integer'}, - 'default_permission_duration_sec': {'type': 'integer'}, - 'info_collection_timeout_sec': {'type': 'integer'}, - 'tenant_limits': copy.deepcopy(CMS_LIMITS_SCHEMA), - 'cluster_limits': copy.deepcopy(CMS_LIMITS_SCHEMA), - 'monitors': { - 'type': 'object', - 'properties': { - 'enable_auto_update': {'type': 'boolean'}, - 'update_interval_sec': {'type': 'integer'}, - 'ignored_downtime_gap_sec': {'type': 'integer'}, - 'broken_timeout_min': {'type': 'integer'}, - 'broken_prep_timeout_min': {'type': 'integer'}, - 'faulty_prep_timeout_min': {'type': 'integer'}, + "type": "object", + "properties": { + "enable_sentinel": dict(type="boolean"), + "default_retry_time_sec": {"type": "integer"}, + "default_permission_duration_sec": {"type": "integer"}, + "info_collection_timeout_sec": {"type": "integer"}, + "tenant_limits": copy.deepcopy(CMS_LIMITS_SCHEMA), + "cluster_limits": copy.deepcopy(CMS_LIMITS_SCHEMA), + "monitors": { + "type": "object", + "properties": { + "enable_auto_update": {"type": "boolean"}, + "update_interval_sec": {"type": "integer"}, + "ignored_downtime_gap_sec": {"type": "integer"}, + "broken_timeout_min": {"type": "integer"}, + "broken_prep_timeout_min": {"type": "integer"}, + "faulty_prep_timeout_min": {"type": "integer"}, }, }, }, } STATE_STORAGE = { - 'type': 'object', - 'properties': { - 'allow_incorrect': {'type': 'boolean'}, - 'node_ids': {'type': 'array', 'items': {'type': 'integer'}}, - 'node_set': {'type': 'array', 'items': {'type': 'string'}}, + "type": "object", + "properties": { + "allow_incorrect": {"type": "boolean"}, + "node_ids": {"type": "array", "items": {"type": "integer"}}, + "node_set": {"type": "array", "items": {"type": "string"}}, }, } -METERING_SCHEMA = {'type': 'object', 'properties': {'metering_file': {'type': 'string'}}} +METERING_SCHEMA = {"type": "object", "properties": {"metering_file": {"type": "string"}}} YQL_SCHEMA = { - 'type': 'object', - 'properties': { - 'endpoint': {'type': 'string'}, - 'database': {'type': 'string'}, - 'table_prefix': {'type': 'string'}, - 'oauth_file': {'type': 'string'}, - 'ydb_mvp_cloud_endpoint': {'type': 'string'}, - 'use_iam_from_metadata_service': {'type': 'boolean'}, - 'use_secure_connection': {'type': 'boolean'}, - 'use_bearer_for_ydb': {'type': 'boolean'}, - 'mdb_gateway': {'type': 'string'}, - 'mdb_transform_host': {'type': 'boolean'}, - 'object_storage_endpoint': {'type': 'string'}, - 'task_service_endpoint': {'type': 'string'}, - 'task_service_database': {'type': 'string'}, - 'impersonation_service_endpoint': {'type': 'string'}, - 'secure_task_service': {'type': 'boolean'}, - 'secure_impersonation_service': {'type': 'boolean'}, - 'hmac_secret_file': {'type': 'string'}, + "type": "object", + "properties": { + "endpoint": {"type": "string"}, + "database": {"type": "string"}, + "table_prefix": {"type": "string"}, + "oauth_file": {"type": "string"}, + "ydb_mvp_cloud_endpoint": {"type": "string"}, + "use_iam_from_metadata_service": {"type": "boolean"}, + "use_secure_connection": {"type": "boolean"}, + "use_bearer_for_ydb": {"type": "boolean"}, + "mdb_gateway": {"type": "string"}, + "mdb_transform_host": {"type": "boolean"}, + "object_storage_endpoint": {"type": "string"}, + "task_service_endpoint": {"type": "string"}, + "task_service_database": {"type": "string"}, + "impersonation_service_endpoint": {"type": "string"}, + "secure_task_service": {"type": "boolean"}, + "secure_impersonation_service": {"type": "boolean"}, + "hmac_secret_file": {"type": "string"}, }, } YQ_SCHEMA = { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'control_plane': { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'endpoint': {'type': 'string'}, - 'database': {'type': 'string'}, - 'table_prefix': {'type': 'string'}, - 'oauth_file': {'type': 'string'}, - 'certificate_file': {'type': 'string'}, - 'iam_endpoint': {'type': 'string'}, - 'sa_key_file': {'type': 'string'}, - 'use_local_metadata_service': {'type': 'boolean'}, - 'enable_forward_analytics': {'type': 'boolean'}, - 'enable_permissions': {'type': 'boolean'}, + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "control_plane": { + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "endpoint": {"type": "string"}, + "database": {"type": "string"}, + "table_prefix": {"type": "string"}, + "oauth_file": {"type": "string"}, + "certificate_file": {"type": "string"}, + "iam_endpoint": {"type": "string"}, + "sa_key_file": {"type": "string"}, + "use_local_metadata_service": {"type": "boolean"}, + "enable_forward_analytics": {"type": "boolean"}, + "enable_permissions": {"type": "boolean"}, }, }, - 'analytics': { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'endpoint': {'type': 'string'}, - 'database': {'type': 'string'}, - 'table_prefix': {'type': 'string'}, - 'oauth_file': {'type': 'string'}, - 'ydb_mvp_cloud_endpoint': {'type': 'string'}, - 'use_iam_from_metadata_service': {'type': 'boolean'}, - 'use_secure_connection': {'type': 'boolean'}, - 'use_bearer_for_ydb': {'type': 'boolean'}, - 'mdb_gateway': {'type': 'string'}, - 'mdb_transform_host': {'type': 'boolean'}, - 'object_storage_endpoint': {'type': 'string'}, - 'task_service_endpoint': {'type': 'string'}, - 'task_service_database': {'type': 'string'}, - 'impersonation_service_endpoint': {'type': 'string'}, - 'secure_task_service': {'type': 'boolean'}, - 'secure_impersonation_service': {'type': 'boolean'}, - 'hmac_secret_file': {'type': 'string'}, + "analytics": { + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "endpoint": {"type": "string"}, + "database": {"type": "string"}, + "table_prefix": {"type": "string"}, + "oauth_file": {"type": "string"}, + "ydb_mvp_cloud_endpoint": {"type": "string"}, + "use_iam_from_metadata_service": {"type": "boolean"}, + "use_secure_connection": {"type": "boolean"}, + "use_bearer_for_ydb": {"type": "boolean"}, + "mdb_gateway": {"type": "string"}, + "mdb_transform_host": {"type": "boolean"}, + "object_storage_endpoint": {"type": "string"}, + "task_service_endpoint": {"type": "string"}, + "task_service_database": {"type": "string"}, + "impersonation_service_endpoint": {"type": "string"}, + "secure_task_service": {"type": "boolean"}, + "secure_impersonation_service": {"type": "boolean"}, + "hmac_secret_file": {"type": "string"}, }, }, - 'streaming': { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'endpoint': {'type': 'string'}, - 'database': {'type': 'string'}, - 'table_prefix': {'type': 'string'}, - 'oauth_file': {'type': 'string'}, - 'certificate_file': {'type': 'string'}, - 'hmac_secret_file': {'type': 'string'}, - 'iam_endpoint': {'type': 'string'}, - 'sa_key_file': {'type': 'string'}, - 'use_local_metadata_service': {'type': 'boolean'}, - 'checkpointing': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'period_millis': {'type': 'integer'}, - 'max_in_flight': {'type': 'integer'}, + "streaming": { + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "endpoint": {"type": "string"}, + "database": {"type": "string"}, + "table_prefix": {"type": "string"}, + "oauth_file": {"type": "string"}, + "certificate_file": {"type": "string"}, + "hmac_secret_file": {"type": "string"}, + "iam_endpoint": {"type": "string"}, + "sa_key_file": {"type": "string"}, + "use_local_metadata_service": {"type": "boolean"}, + "checkpointing": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "period_millis": {"type": "integer"}, + "max_in_flight": {"type": "integer"}, }, }, }, }, - 'token_accessor': { - 'type': 'object', - 'properties': {'endpoint': {'type': 'string'}, 'use_ssl': {'type': 'boolean'}}, + "token_accessor": { + "type": "object", + "properties": {"endpoint": {"type": "string"}, "use_ssl": {"type": "boolean"}}, }, - 'folder_service': { - 'type': 'object', - 'properties': { - 'enable': {'type': 'boolean'}, - 'endpoint': {'type': 'string'}, - 'path_to_root_ca': {'type': 'string'}, + "folder_service": { + "type": "object", + "properties": { + "enable": {"type": "boolean"}, + "endpoint": {"type": "string"}, + "path_to_root_ca": {"type": "string"}, }, }, }, @@ -830,54 +830,54 @@ YQ_SCHEMA = { TEMPLATE_SCHEMA = { "type": "object", "properties": { - 'enable_cores': dict(type='boolean'), - 'state_storage': copy.deepcopy(STATE_STORAGE), - "system_tablets": {'type': 'object'}, - "forbid_implicit_storage_pools": {'type': 'boolean'}, - "use_fixed_tablet_types": {'type': 'boolean'}, - "monitoring_address": {'type': 'string'}, - "use_console_feature": {'type': 'boolean'}, - 'storage_config_generation': {'type': 'integer'}, - 'use_walle': {'type': 'boolean'}, - 'cloud_mode': {'type': 'boolean'}, - 'host_configs': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'generation': {'type': 'integer', 'min': 0}, - 'drives': { - 'type': 'array', - 'items': copy.deepcopy(DRIVE_SCHEMA), + "enable_cores": dict(type="boolean"), + "state_storage": copy.deepcopy(STATE_STORAGE), + "system_tablets": {"type": "object"}, + "forbid_implicit_storage_pools": {"type": "boolean"}, + "use_fixed_tablet_types": {"type": "boolean"}, + "monitoring_address": {"type": "string"}, + "use_console_feature": {"type": "boolean"}, + "storage_config_generation": {"type": "integer"}, + "use_walle": {"type": "boolean"}, + "cloud_mode": {"type": "boolean"}, + "host_configs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "generation": {"type": "integer", "min": 0}, + "drives": { + "type": "array", + "items": copy.deepcopy(DRIVE_SCHEMA), }, }, }, }, - 'static_bs_group_hosts': { - 'type': 'array', - 'items': { - 'type': 'string', + "static_bs_group_hosts": { + "type": "array", + "items": { + "type": "string", }, }, - 'use_cluster_uuid': { - 'type': 'boolean', + "use_cluster_uuid": { + "type": "boolean", }, - 'require_address': { - 'type': 'boolean', + "require_address": { + "type": "boolean", }, - 'cluster_uuid': { - 'type': 'string', - 'minLength': 1, + "cluster_uuid": { + "type": "string", + "minLength": 1, }, - 'accepted_cluster_uuids': {'type': 'array', 'items': {'type': 'string', 'minLength': 1}}, - 'security_settings': { - 'type': 'object', - 'properties': { - 'enforce_user_token_requirement': { - 'type': 'boolean', + "accepted_cluster_uuids": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "security_settings": { + "type": "object", + "properties": { + "enforce_user_token_requirement": { + "type": "boolean", }, - 'monitoring_allowed_sids': {'type': 'array', 'items': {'type': 'string'}}, - 'administration_allowed_sids': {'type': 'array', 'items': {'type': 'string'}}, + "monitoring_allowed_sids": {"type": "array", "items": {"type": "string"}}, + "administration_allowed_sids": {"type": "array", "items": {"type": "string"}}, }, }, "static_erasure": { @@ -890,58 +890,58 @@ TEMPLATE_SCHEMA = { "minLength": 1, "enum": FailDomainType.all_fail_domain_type_names(), }, - 'static_pdisk_type': { - 'type': 'string', - 'enum': PDiskCategory.all_pdisk_category_names(), + "static_pdisk_type": { + "type": "string", + "enum": PDiskCategory.all_pdisk_category_names(), }, - 'nw_cache_file_path': { - 'type': 'string', + "nw_cache_file_path": { + "type": "string", }, - 'enable_cms_config_cache': { - 'type': 'boolean', + "enable_cms_config_cache": { + "type": "boolean", }, "hosts": { "type": "array", "items": copy.deepcopy(HOST_SCHEMA), - 'minItems': 1, - 'checkNameServiceDuplicates': True, - }, - 'auth': {'type': 'object'}, - 'log': copy.deepcopy(LOG_SCHEMA), - 'grpc': {'type': 'object'}, - 'kqp': copy.deepcopy(KQP_SCHEMA), - 'ic': {'type': 'object'}, - 'pq': {'type': 'object'}, - 'storage_pools': {'type': 'array', 'items': copy.deepcopy(STORAGE_POOL)}, - 'profiles': copy.deepcopy(PROFILES), - 'domains': {'type': 'array', 'items': copy.deepcopy(DOMAIN_SCHEMA), 'minItems': 1}, - 'nbs': copy.deepcopy(NBS_SCHEMA), - 'nbs_control': copy.deepcopy(NBS_SCHEMA), - 'nfs': copy.deepcopy(NFS_SCHEMA), - 'nfs_control': copy.deepcopy(NFS_SCHEMA), - 'sqs': copy.deepcopy(SQS_SCHEMA), - 'features': copy.deepcopy(FEATURES_SCHEMA), - 'shared_cache': copy.deepcopy(SHARED_CACHE_SCHEMA), - 'sys': copy.deepcopy(SYS_SCHEMA), - 'tracing': copy.deepcopy(TRACING_SCHEMA), - 'failure_injection_config': copy.deepcopy(FAILURE_INJECTION_CONFIG_SCHEMA), - 'solomon': copy.deepcopy(SOLOMON_SCHEMA), - 'cms': copy.deepcopy(CMS_SCHEMA), - 'resource_broker': {'type': 'object'}, - 'state_storages': { - 'type': 'array', - 'items': {'type': 'object', 'properties': {}, 'additionalProperties': True}, - }, - 'metering': copy.deepcopy(METERING_SCHEMA), - 'yql_analytics': copy.deepcopy(YQL_SCHEMA), - 'yq': copy.deepcopy(YQ_SCHEMA), + "minItems": 1, + "checkNameServiceDuplicates": True, + }, + "auth": {"type": "object"}, + "log": copy.deepcopy(LOG_SCHEMA), + "grpc": {"type": "object"}, + "kqp": copy.deepcopy(KQP_SCHEMA), + "ic": {"type": "object"}, + "pq": {"type": "object"}, + "storage_pools": {"type": "array", "items": copy.deepcopy(STORAGE_POOL)}, + "profiles": copy.deepcopy(PROFILES), + "domains": {"type": "array", "items": copy.deepcopy(DOMAIN_SCHEMA), "minItems": 1}, + "nbs": copy.deepcopy(NBS_SCHEMA), + "nbs_control": copy.deepcopy(NBS_SCHEMA), + "nfs": copy.deepcopy(NFS_SCHEMA), + "nfs_control": copy.deepcopy(NFS_SCHEMA), + "sqs": copy.deepcopy(SQS_SCHEMA), + "features": copy.deepcopy(FEATURES_SCHEMA), + "shared_cache": copy.deepcopy(SHARED_CACHE_SCHEMA), + "sys": copy.deepcopy(SYS_SCHEMA), + "tracing": copy.deepcopy(TRACING_SCHEMA), + "failure_injection_config": copy.deepcopy(FAILURE_INJECTION_CONFIG_SCHEMA), + "solomon": copy.deepcopy(SOLOMON_SCHEMA), + "cms": copy.deepcopy(CMS_SCHEMA), + "resource_broker": {"type": "object"}, + "state_storages": { + "type": "array", + "items": {"type": "object", "properties": {}, "additionalProperties": True}, + }, + "metering": copy.deepcopy(METERING_SCHEMA), + "yql_analytics": copy.deepcopy(YQL_SCHEMA), + "yq": copy.deepcopy(YQ_SCHEMA), }, "required": ["static_erasure", "hosts"], } def _host_and_ic_port(host): - return "%s:%s" % (host['name'], str(host.get('ic_port', 19001))) + return "%s:%s" % (host["name"], str(host.get("ic_port", 19001))) def checkNameServiceDuplicates(validator, allow_duplicates, instance, schema): @@ -962,7 +962,7 @@ _Validator = jsonschema.Draft4Validator _Validator = jsonschema.validators.extend( _Validator, { - 'checkNameServiceDuplicates': checkNameServiceDuplicates, + "checkNameServiceDuplicates": checkNameServiceDuplicates, }, ) @@ -973,6 +973,9 @@ class Validator(_Validator): super(Validator, self).__init__(schema, format_checker=format_checker) +def default_validator(): + return Validator(copy.deepcopy(TEMPLATE_SCHEMA)) + + def validate(template): - schema = copy.deepcopy(TEMPLATE_SCHEMA) - Validator(schema).validate(template) + default_validator().validate(template) diff --git a/ydb/tools/cfg/walle/__init__.py b/ydb/tools/cfg/walle/__init__.py index df7d52021c..808bc9e2bd 100644 --- a/ydb/tools/cfg/walle/__init__.py +++ b/ydb/tools/cfg/walle/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from .walle import HostsFetcherInterface, NopHostsFetcher # noqa +from .walle import HostsInformationProvider, NopHostsInformationProvider, WalleHostsInformationProvider # noqa -__all__ = ('Walle',) +__all__ = ("Walle",) diff --git a/ydb/tools/cfg/walle/walle.py b/ydb/tools/cfg/walle/walle.py index 92792aaf6e..e2677f48a9 100644 --- a/ydb/tools/cfg/walle/walle.py +++ b/ydb/tools/cfg/walle/walle.py @@ -1,10 +1,19 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import os +import requests +import threading import zlib +import json +import time + + from abc import ABCMeta, abstractmethod +from six.moves.urllib import parse + -class HostsFetcherInterface: +class HostsInformationProvider: __metaclass__ = ABCMeta @abstractmethod @@ -20,7 +29,7 @@ class HostsFetcherInterface: pass -class NopHostsFetcher(HostsFetcherInterface): +class NopHostsInformationProvider(HostsInformationProvider): def __init__(self): pass @@ -32,3 +41,65 @@ class NopHostsFetcher(HostsFetcherInterface): def get_body(self, hostname): return zlib.crc32(hostname.encode()) + + +class WalleHostsInformationProvider(HostsInformationProvider): + def __init__(self, provider_url=None, cloud_mode=False): + if provider_url is None: + raise RuntimeError("Got empty hosts_provider url") + self._base_url = provider_url + self._cache = {} + self._timeout_seconds = 5 + self._retry_count = 10 + self._cloud_mode = cloud_mode + self._lock = threading.Lock() + + def _ask_location(self, hostname): + with self._lock: + if hostname in self._cache: + return self._cache[hostname] + + with requests.Session() as session: + retries = 10 + url = self._base_url + hostname + "?" + parse.urlencode({"fields": "location,name,inv,short_queue_name"}) + headers = dict() + if "YC_TOKEN" in os.environ: + headers["Authorization"] = "Bearer " + os.environ.get("YC_TOKEN").strip() + + while retries > 0: + retries -= 1 + + try: + response = session.get(url, timeout=self._timeout_seconds, headers=headers) + response.raise_for_status() + loaded = json.loads(response.content) + with self._lock: + self._cache[hostname] = loaded + return self._cache[hostname] + + except IOError: + if retries == 0: + raise RuntimeError( + "Failed to retrieve information about host %s in %d retries" + % ( + hostname, + self._retry_count, + ) + ) + + time.sleep(3) + raise RuntimeError() + + def get_rack(self, hostname): + short_queue_name = self._ask_location(hostname)["location"]["short_queue_name"] + rack = self._ask_location(hostname)["location"]["rack"] + if self._cloud_mode: + return "{}:{}".format(short_queue_name.lower(), rack.lower()) + return "{}#{}".format(short_queue_name.upper(), rack) + + def get_datacenter(self, hostname): + short_dc_name = self._ask_location(hostname)["location"]["short_datacenter_name"] + return short_dc_name if self._cloud_mode else short_dc_name.upper() + + def get_body(self, hostname): + return self._ask_location(hostname)["inv"] |