diff options
author | jansenin <jansenin@yandex-team.com> | 2023-10-03 11:25:50 +0300 |
---|---|---|
committer | jansenin <jansenin@yandex-team.com> | 2023-10-03 12:05:57 +0300 |
commit | d6e6ee8dea5c7a1854889fb47138c835a1693a2a (patch) | |
tree | 31cf1d59d21e6a9468ef611c8ecaf19fa695d449 | |
parent | fc96d0ea409e2b5d8ed1f2eef83af992fb5d5822 (diff) | |
download | ydb-d6e6ee8dea5c7a1854889fb47138c835a1693a2a.tar.gz |
PR from branch users/jansenin/static-config-validation2
tests for static validator
add static validator builder part and make make enums case-insensitive
19 files changed, 1481 insertions, 58 deletions
diff --git a/.mapping.json b/.mapping.json index 370dc51695..7c0ee71c47 100644 --- a/.mapping.json +++ b/.mapping.json @@ -6624,6 +6624,11 @@ "ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-x86_64.txt":"", "ydb/library/yaml_config/static_validator/ut/CMakeLists.txt":"", "ydb/library/yaml_config/static_validator/ut/CMakeLists.windows-x86_64.txt":"", + "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.darwin-x86_64.txt":"", + "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-aarch64.txt":"", + "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-x86_64.txt":"", + "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.txt":"", + "ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.windows-x86_64.txt":"", "ydb/library/yaml_config/ut/CMakeLists.darwin-x86_64.txt":"", "ydb/library/yaml_config/ut/CMakeLists.linux-aarch64.txt":"", "ydb/library/yaml_config/ut/CMakeLists.linux-x86_64.txt":"", diff --git a/ydb/library/yaml_config/static_validator/builders.cpp b/ydb/library/yaml_config/static_validator/builders.cpp index e38ff01938..64f778530d 100644 --- a/ydb/library/yaml_config/static_validator/builders.cpp +++ b/ydb/library/yaml_config/static_validator/builders.cpp @@ -7,13 +7,17 @@ using namespace NYamlConfig::NValidator; using namespace NYamlConfig::NValidator::Configurators; TArrayBuilder HostConfigBuilder() { - return TArrayBuilder([](auto& b) {b + return TArrayBuilder([](auto& hostsConfig) { + hostsConfig .Configure(uniqueByPath("host_config_id")) - .MapItem([](auto& b) {b + .MapItem([](auto& host) { + host .Int64("host_config_id", nonNegative()) // >= 0? - .Array("drive", [](auto& b) {b + .Array("drive", [](auto& drive) { + drive .Configure(uniqueByPath("path")) - .MapItem([](auto& b) {b + .MapItem([](auto& driveItem) { + driveItem .String("path") .Enum("type", {"ssd", "nvme", "rot"}); }); @@ -23,33 +27,38 @@ TArrayBuilder HostConfigBuilder() { } TArrayBuilder HostsBuilder() { - return TArrayBuilder([](auto& b){b - .MapItem([](auto& b) {b + return TArrayBuilder([](auto& hosts){ + hosts + .MapItem([](auto& host) { + host .String("host") .Int64("host_config_id", nonNegative()) - .Int64("node_id", nonNegative()) - .Int64("port", 0, 65535) - .Map("location", [](auto& b){b - .String("unit") - .String("data_center") - .String("rack"); + .String("node_id", [](auto& nodeId){ + nodeId.Optional(); + }) + .Int64("port", [](auto& port){ + port + .Range(0, 65535) + .Optional(); }); }) .Configure(uniqueByPath("node_id")) - .AddCheck("Must not have to hosts with same host name and port", [](auto& c) { - auto array = c.Node(); + .AddCheck("Must not have two hosts with same host name and port", [](auto& hostsContext) { + auto array = hostsContext.Node(); THashMap<std::pair<TString, i64>, i64> itemsToIndex; for (int i = 0; i < array.Length(); ++i) { TString host = array[i].Map()["host"].String(); - i64 port = array[i].Map()["port"].Int64(); + auto portNode = array[i].Map()["port"]; + i64 defaultPort = 19001; + i64 port = portNode.Exists() ? portNode.Int64() : defaultPort; auto item = std::pair{host, port}; if (itemsToIndex.contains(item)) { TString i1 = ToString(itemsToIndex[item]); TString i2 = ToString(i); - c.Fail("items with indexes " + i1 + " and " + i2 + " are conflicting"); + hostsContext.Fail("items with indexes " + i1 + " and " + i2 + " are conflicting"); } itemsToIndex[item] = i; } @@ -58,9 +67,12 @@ TArrayBuilder HostsBuilder() { } TMapBuilder DomainsConfigBuilder() { - return TMapBuilder([](auto& b){b - .Array("domain", [](auto& b){b - .MapItem([](auto& b){b + return TMapBuilder([](auto& domainsConfig){ + domainsConfig + .Array("domain", [](auto& domain){ + domain + .MapItem([](auto& domainItem){ + domainItem .String("name") .Field("storage_pool_types", StoragePoolTypesConfigBuilder()); }); @@ -71,44 +83,58 @@ TMapBuilder DomainsConfigBuilder() { } TArrayBuilder StoragePoolTypesConfigBuilder() { - return TArrayBuilder([](auto& b){b - .MapItem([](auto& b){b + return TArrayBuilder([](auto& storagePoolTypes){ + storagePoolTypes + .MapItem([](auto& storagePoolType){ + storagePoolType .String("kind") - .Map("pool_config", [](auto& b){b + .Map("pool_config", [](auto& poolConfig){ + poolConfig .Int64("box_id", nonNegative()) - .Int64("encryption", [](auto&b){b + .Int64("encryption", [](auto&encryption){ + encryption .Range(0, 1) .Optional(); }) .Enum("erasure_species", {"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"}) .String("kind") - .Array("pdisk_filter", [](auto& b){b - .MapItem([](auto& b){b - .Array("property", [](auto& b){b - .MapItem([](auto& b){b + .Array("pdisk_filter", [](auto& pdiskFilter){ + pdiskFilter + .MapItem([](auto& pdiskFilterItem){ + pdiskFilterItem + .Array("property", [](auto& property){ + property + .MapItem([](auto& propertyItem){ + propertyItem .Enum("type", {"ssd", "nvme", "rot"}); }); }); }); }); - }) - .Configure(mustBeEqual("kind", "pool_config/kind")); + }); + storagePoolType.Configure(mustBeEqual("kind", "pool_config/kind")); }); }); } TArrayBuilder StateStorageBuilder() { - return TArrayBuilder([](auto& b){b - .MapItem([](auto& b){b - .Map("ring", [](auto& b){b - .Array("node", [](auto& b){b - .Int64Item(nonNegative()); + return TArrayBuilder([](auto& stateStorage){ + stateStorage + .MapItem([](auto& stateStorageItem){ + stateStorageItem + .Map("ring", [](auto& ring){ + ring + .Array("node", [](auto& node){ + node.Int64Item(nonNegative()); }) .Int64("nto_select", nonNegative()) - .AddCheck("nto_select must not be greater, than node array size", [](auto& c){ - i64 nto_select = c.Node()["nto_select"].Int64(); - i64 arr_size = c.Node()["node"].Array().Length(); - c.Expect(nto_select <= arr_size); + .AddCheck("nto_select must not be greater, than node array size", [](auto& ringContext){ + i64 nto_select = ringContext.Node()["nto_select"].Int64(); + i64 arr_size = ringContext.Node()["node"].Array().Length(); + ringContext.Expect(nto_select <= arr_size); + }) + .AddCheck("nto_select should be odd, because even number usage doesn't improve fault tolerance", [](auto& ringContext){ + ringContext.Expect(ringContext.Node()["nto_select"].Int64() % 2 == 1); }); }) .Int64("ssid", nonNegative()); @@ -117,10 +143,168 @@ TArrayBuilder StateStorageBuilder() { } TMapBuilder SecurityConfigBuilder() { - return TMapBuilder([](auto& b){b - .Bool("enforce_user_token_requirement", [](auto& b){ - b.Optional(); + return TMapBuilder([](auto& securityConfig){ + securityConfig + .Bool("enforce_user_token_requirement", [](auto& enforceUserTokenRequirements){ + enforceUserTokenRequirements.Optional(); }) .Optional(); }); } + +TMapBuilder ActorSystemConfigBuilder() { + return TMapBuilder([](auto& actorSystem){ + actorSystem + .Bool("use_auto_config", [](auto& useAutoConfig){ + useAutoConfig.Optional(); + }) + .Enum("node_type", [](auto& nodeType){ + nodeType + .SetItems({"STORAGE", "COMPUTE", "HYBRID"}) + .Optional(); + }) + .Int64("cpu_count", [](auto& cpuCount){ + cpuCount + .Min(0) + .Optional(); + }) + .Array("executor", [](auto& executor){ + executor + .Optional() + .MapItem([](auto& executorItem){ + executorItem + .Enum("name", {"System", "User", "Batch", "IO", "IC"}) + .Int64("spin_threshold", [](auto& spinThreshold){ + spinThreshold + .Min(0) + .Optional(); + }) + .Int64("threads", nonNegative()) + .Int64("max_threads", [](auto& maxThreads){ + maxThreads + .Optional() + .Min(0); + }) + .Int64("max_avg_ping_deviation", [](auto& maxAvgPingDeviation){ + maxAvgPingDeviation + .Min(0) + .Optional(); + }) + .Int64("time_per_mailbox_micro_secs", [](auto& timePerMailboxMicroSecs){ + timePerMailboxMicroSecs + .Optional() + .Min(0); + }) + .Enum("type", {"IO", "BASIC"}); + }); + }) + .Map("scheduler", [](auto& scheduler){ + scheduler + .Optional() + .Int64("progress_threshold", nonNegative()) + .Int64("resolution", nonNegative()) + .Int64("spin_threshold", nonNegative()); + }) + .AddCheck("Must either be auto config or manual config", [](auto& actorSystemContext){ + bool autoConfig = false; + auto node = actorSystemContext.Node(); + if (node["use_auto_config"].Exists() && node["use_auto_config"].Bool()) { + autoConfig = true; + } + if (autoConfig) { + actorSystemContext.Expect(node["node_type"].Exists(), "node_type must exist when using auto congfig"); + actorSystemContext.Expect(node["cpu_count"].Exists(), "cpu_count must exist when using auto congfig"); + + actorSystemContext.Expect(!node["executor"].Exists(), "executor must not exist when using auto congfig"); + actorSystemContext.Expect(!node["scheduler"].Exists(), "scheduler must not exist when using auto congfig"); + } else { + actorSystemContext.Expect(node["executor"].Exists(), "executor must exist when not using auto congfig"); + actorSystemContext.Expect(node["scheduler"].Exists(), "scheduler must exist when not using auto congfig"); + + actorSystemContext.Expect(!node["node_type"].Exists(), "node_type must not exist when not using auto congfig"); + actorSystemContext.Expect(!node["cpu_count"].Exists(), "cpu_count must not exist when not using auto congfig"); + } + }); + }); +} + +NYamlConfig::NValidator::TMapBuilder BlobStorageConfigBuilder() { + return TMapBuilder([](auto& blobStorageConfig){ + blobStorageConfig + .Map("service_set", [](auto& serviceSet){ + serviceSet + .Array("groups", [](auto& groups){ + groups + .MapItem([](auto& group){ + group + .Enum("erasure_species", {"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"}) + .Array("rings", [](auto& rings){ + rings + .MapItem([](auto& ring){ + ring + .Array("fail_domains", [](auto& failDomains){ + failDomains + .MapItem([](auto& failDomain){ + failDomain + .Array("vdisk_locations", [](auto& vdiskLocations){ + vdiskLocations + .MapItem([](auto& vdiskLocation){ + vdiskLocation + .String("node_id") + .String("path") + .Enum("pdisk_category", {"ssd", "nvme", "rot"}); + }); + }); + }); + }); + }); + }); + }); + }); + }); + }); +} + +NYamlConfig::NValidator::TMapBuilder ChannelProfileConfigBuilder() { + return TMapBuilder([](auto& channelProfileConfig){ + channelProfileConfig + .Array("profile", [](auto& profile){ + profile + .MapItem([](auto& profileItem){ + profileItem + .Array("channel", [](auto& channel){ + channel + .MapItem([](auto& channelItem){ + channelItem + .Enum("erasure_species", {"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"}) + .Int64("pdisk_category") + .Enum("storage_pool_kind", {"nvme", "ssd", "rot"}); + }); + }) + .Int64("profile_id", nonNegative()); + }); + }); + }); +} + +NYamlConfig::NValidator::TMapBuilder StaticConfigBuilder() { + return TMapBuilder([](auto& staticConfig){ + staticConfig + .Enum("static_erasure", [](auto& staticErasure){ + staticErasure + .SetItems({"none", "block-4-2", "mirror-3-dc", "mirror-3dc-3-nodes"}) + .Optional(); + }) + .Field("host_configs", HostConfigBuilder()) + .Field("hosts", HostsBuilder()) + .Field("domains_config", DomainsConfigBuilder()) + .Map("table_service_config", [](auto& tableServiceConfig){ + tableServiceConfig + .Optional() + .String("sql_version"); + }) + .Field("actor_system_config", ActorSystemConfigBuilder()) + .Field("blob_storage_config", BlobStorageConfigBuilder()) + .Field("channel_profile_config", ChannelProfileConfigBuilder()); + }); +} diff --git a/ydb/library/yaml_config/static_validator/builders.h b/ydb/library/yaml_config/static_validator/builders.h index ec81eaa42e..aaa3a672d3 100644 --- a/ydb/library/yaml_config/static_validator/builders.h +++ b/ydb/library/yaml_config/static_validator/builders.h @@ -7,3 +7,10 @@ NYamlConfig::NValidator::TMapBuilder DomainsConfigBuilder(); NYamlConfig::NValidator::TArrayBuilder StoragePoolTypesConfigBuilder(); NYamlConfig::NValidator::TArrayBuilder StateStorageBuilder(); NYamlConfig::NValidator::TMapBuilder SecurityConfigBuilder(); + +NYamlConfig::NValidator::TMapBuilder ActorSystemConfigBuilder(); +NYamlConfig::NValidator::TMapBuilder BlobStorageConfigBuilder(); + +NYamlConfig::NValidator::TMapBuilder ChannelProfileConfigBuilder(); + +NYamlConfig::NValidator::TMapBuilder StaticConfigBuilder(); diff --git a/ydb/library/yaml_config/static_validator/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yaml_config/static_validator/ut/CMakeLists.darwin-x86_64.txt index 5ba673b5a0..83e5fe99d1 100644 --- a/ydb/library/yaml_config/static_validator/ut/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yaml_config/static_validator/ut/CMakeLists.darwin-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(example_configs) add_executable(ydb-library-yaml_config-static_validator-ut) target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC diff --git a/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-aarch64.txt index 2cc2a1d91e..f943c0a962 100644 --- a/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-aarch64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(example_configs) add_executable(ydb-library-yaml_config-static_validator-ut) target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC diff --git a/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-x86_64.txt index ab86023928..8de063feaa 100644 --- a/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yaml_config/static_validator/ut/CMakeLists.linux-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(example_configs) add_executable(ydb-library-yaml_config-static_validator-ut) target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC diff --git a/ydb/library/yaml_config/static_validator/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yaml_config/static_validator/ut/CMakeLists.windows-x86_64.txt index 749e97fd3c..68abb35441 100644 --- a/ydb/library/yaml_config/static_validator/ut/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yaml_config/static_validator/ut/CMakeLists.windows-x86_64.txt @@ -6,6 +6,7 @@ # original buildsystem will not be accepted. +add_subdirectory(example_configs) add_executable(ydb-library-yaml_config-static_validator-ut) target_link_libraries(ydb-library-yaml_config-static_validator-ut PUBLIC diff --git a/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.darwin-x86_64.txt b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..cfdca304d9 --- /dev/null +++ b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,63 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(static_validator-ut-example_configs) +target_link_libraries(static_validator-ut-example_configs PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + library-yaml_config-static_validator + library-yaml_config-validator +) +target_link_options(static_validator-ut-example_configs PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC +) +target_sources(static_validator-ut-example_configs PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp +) +set_property( + TARGET + static_validator-ut-example_configs + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + static_validator-ut-example_configs + TEST_TARGET + static_validator-ut-example_configs + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + PROCESSORS + 1 +) +target_allocator(static_validator-ut-example_configs + system_allocator +) +vcs_info(static_validator-ut-example_configs) diff --git a/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-aarch64.txt b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..f5e109052a --- /dev/null +++ b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-aarch64.txt @@ -0,0 +1,68 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(static_validator-ut-example_configs) +target_link_libraries(static_validator-ut-example_configs PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-testing-unittest_main + library-yaml_config-static_validator + library-yaml_config-validator +) +target_link_options(static_validator-ut-example_configs PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(static_validator-ut-example_configs PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp +) +set_property( + TARGET + static_validator-ut-example_configs + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + static_validator-ut-example_configs + TEST_TARGET + static_validator-ut-example_configs + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + PROCESSORS + 1 +) +target_allocator(static_validator-ut-example_configs + cpp-malloc-jemalloc +) +vcs_info(static_validator-ut-example_configs) diff --git a/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-x86_64.txt b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..f75721a24e --- /dev/null +++ b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.linux-x86_64.txt @@ -0,0 +1,70 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(static_validator-ut-example_configs) +target_link_libraries(static_validator-ut-example_configs PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + library-yaml_config-static_validator + library-yaml_config-validator +) +target_link_options(static_validator-ut-example_configs PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(static_validator-ut-example_configs PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp +) +set_property( + TARGET + static_validator-ut-example_configs + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + static_validator-ut-example_configs + TEST_TARGET + static_validator-ut-example_configs + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + PROCESSORS + 1 +) +target_allocator(static_validator-ut-example_configs + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(static_validator-ut-example_configs) diff --git a/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.txt b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.windows-x86_64.txt b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..8521e7c92f --- /dev/null +++ b/ydb/library/yaml_config/static_validator/ut/example_configs/CMakeLists.windows-x86_64.txt @@ -0,0 +1,58 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_executable(static_validator-ut-example_configs) +target_link_libraries(static_validator-ut-example_configs PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + cpp-testing-unittest_main + library-yaml_config-static_validator + library-yaml_config-validator +) +target_sources(static_validator-ut-example_configs PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp +) +set_property( + TARGET + static_validator-ut-example_configs + PROPERTY + SPLIT_FACTOR + 1 +) +add_yunittest( + NAME + static_validator-ut-example_configs + TEST_TARGET + static_validator-ut-example_configs + TEST_ARG + --print-before-suite + --print-before-test + --fork-tests + --print-times + --show-fails +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + LABELS + SMALL +) +set_yunittest_property( + TEST + static_validator-ut-example_configs + PROPERTY + PROCESSORS + 1 +) +target_allocator(static_validator-ut-example_configs + system_allocator +) +vcs_info(static_validator-ut-example_configs) diff --git a/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp b/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp new file mode 100644 index 0000000000..0cda850afb --- /dev/null +++ b/ydb/library/yaml_config/static_validator/ut/example_configs/test.cpp @@ -0,0 +1,907 @@ +#include <ydb/library/yaml_config/static_validator/builders.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <ydb/library/yaml_config/validator/validator.h> +#include <ydb/library/yaml_config/validator/validator_builder.h> +#include <ydb/library/yaml_config/static_validator/builders.h> + +using namespace NYamlConfig::NValidator; +using TIssue = TValidationResult::TIssue; + +bool HasOnlyThisIssues(TValidationResult result, TVector<TIssue> issues) { + if (result.Issues.size() != issues.size()) { + Cerr << "Issue counts are differend. List of actul issues:" << Endl; + Cerr << result; + Cerr << "------------- List of Expected Issues: " << Endl; + Cerr << TValidationResult(issues); + Cerr << "------------- End of issue List" << Endl; + return false; + } + Sort(result.Issues); + Sort(issues); + for (size_t i = 0; i < issues.size(); ++i) { + if (result.Issues[i] != issues[i]) { + Cerr << "Issues are differend. List of actul issues:" << Endl; + Cerr << result; + Cerr << "------------- List of Expected Issues: " << Endl; + Cerr << TValidationResult(issues); + Cerr << "------------- End of issue List" << Endl; + return false; + } + } + return true; +} + +bool Valid(TValidationResult result) { + if (result.Ok()) return true; + + Cerr << "List of issues:" << Endl; + Cerr << result; + Cerr << "------------- End of issue list: " << Endl; + return false; +} + +Y_UNIT_TEST_SUITE(StaticConfigExamples) { + Y_UNIT_TEST(SingleNodeWithFile) { + auto v = StaticConfigBuilder().CreateValidator(); + + const char* yaml = + "static_erasure: none\n" + "host_configs:\n" + "- drive:\n" + " - path: /tmp/pdisk.data\n" + " type: SSD\n" + " host_config_id: 1\n" + "hosts:\n" + "- host: localhost\n" + " host_config_id: 1\n" + " port: 19001\n" + " walle_location:\n" + " body: 1\n" + " data_center: '1'\n" + " rack: '1'\n" + "domains_config:\n" + " domain:\n" + " - name: Root\n" + " storage_pool_types:\n" + " - kind: ssd\n" + " pool_config:\n" + " box_id: 1\n" + " erasure_species: none\n" + " kind: ssd\n" + " pdisk_filter:\n" + " - property:\n" + " - type: SSD\n" + " vdisk_kind: Default\n" + " state_storage:\n" + " - ring:\n" + " node:\n" + " - 1\n" + " nto_select: 1\n" + " ssid: 1\n" + "table_service_config:\n" + " sql_version: 1\n" + "actor_system_config:\n" + " executor:\n" + " - name: System\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: User\n" + " spin_threshold: 0\n" + " threads: 3\n" + " type: BASIC\n" + " - name: Batch\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: IO\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: IO\n" + " - name: IC\n" + " spin_threshold: 10\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: BASIC\n" + " scheduler:\n" + " progress_threshold: 10000\n" + " resolution: 256\n" + " spin_threshold: 0\n" + "blob_storage_config:\n" + " service_set:\n" + " groups:\n" + " - erasure_species: none\n" + " rings:\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: 1\n" + " path: /tmp/pdisk.data\n" + " pdisk_category: SSD\n" + "channel_profile_config:\n" + " profile:\n" + " - channel:\n" + " - erasure_species: none\n" + " pdisk_category: 0\n" + " storage_pool_kind: ssd\n" + " - erasure_species: none\n" + " pdisk_category: 0\n" + " storage_pool_kind: ssd\n" + " - erasure_species: none\n" + " pdisk_category: 0\n" + " storage_pool_kind: ssd\n" + " profile_id: 0\n"; + + Y_ENSURE(Valid(v.Validate(yaml))); + } + + Y_UNIT_TEST(BLOCK42) { + auto v = StaticConfigBuilder().CreateValidator(); + const char* yaml = + "static_erasure: block-4-2\n" + "host_configs:\n" + "- drive:\n" + " - path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " type: SSD\n" + " - path: /dev/disk/by-partlabel/ydb_disk_ssd_02\n" + " type: SSD\n" + " host_config_id: 1\n" + "hosts:\n" + "- host: ydb-node-zone-a-1.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 1\n" + " data_center: \"zone-a\"\n" + " rack: \"1\"\n" + "- host: ydb-node-zone-a-2.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 2\n" + " data_center: \"zone-a\"\n" + " rack: \"2\"\n" + "- host: ydb-node-zone-a-3.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 3\n" + " data_center: \"zone-a\"\n" + " rack: \"3\"\n" + "\n" + "- host: ydb-node-zone-a-4.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 4\n" + " data_center: \"zone-a\"\n" + " rack: \"4\"\n" + "- host: ydb-node-zone-a-5.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 5\n" + " data_center: \"zone-a\"\n" + " rack: \"5\"\n" + "- host: ydb-node-zone-a-6.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 6\n" + " data_center: \"zone-a\"\n" + " rack: \"6\"\n" + "\n" + "- host: ydb-node-zone-a-7.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 7\n" + " data_center: \"zone-a\"\n" + " rack: \"7\"\n" + "- host: ydb-node-zone-a-8.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 8\n" + " data_center: \"zone-a\"\n" + " rack: \"8\"\n" + "\n" + "domains_config:\n" + " domain:\n" + " - name: Root\n" + " storage_pool_types:\n" + " - kind: ssd\n" + " pool_config:\n" + " box_id: 1\n" + " erasure_species: block-4-2\n" + " kind: ssd\n" + " pdisk_filter:\n" + " - property:\n" + " - type: SSD\n" + " vdisk_kind: Default\n" + " state_storage:\n" + " - ring:\n" + " node: [1, 2, 3, 4, 5, 6, 7, 8]\n" + " nto_select: 5\n" + " ssid: 1\n" + "table_service_config:\n" + " sql_version: 1\n" + "actor_system_config:\n" + " executor:\n" + " - name: System\n" + " threads: 2\n" + " type: BASIC\n" + " - name: User\n" + " threads: 3\n" + " type: BASIC\n" + " - name: Batch\n" + " threads: 2\n" + " type: BASIC\n" + " - name: IO\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: IO\n" + " - name: IC\n" + " spin_threshold: 10\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: BASIC\n" + " scheduler:\n" + " progress_threshold: 10000\n" + " resolution: 256\n" + " spin_threshold: 0\n" + "blob_storage_config:\n" + " service_set:\n" + " groups:\n" + " - erasure_species: block-4-2\n" + " rings:\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-1.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-2.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-3.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-4.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-5.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-6.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-7.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-8.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + "channel_profile_config:\n" + " profile:\n" + " - channel:\n" + " - erasure_species: block-4-2\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: block-4-2\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: block-4-2\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " profile_id: 0\n" + "interconnect_config:\n" + " start_tcp: true\n" + " encryption_mode: OPTIONAL\n" + " path_to_certificate_file: \"/opt/ydb/certs/node.crt\"\n" + " path_to_private_key_file: \"/opt/ydb/certs/node.key\"\n" + " path_to_ca_file: \"/opt/ydb/certs/ca.crt\"\n" + "grpc_config:\n" + " cert: \"/opt/ydb/certs/node.crt\"\n" + " key: \"/opt/ydb/certs/node.key\"\n" + " ca: \"/opt/ydb/certs/ca.crt\"\n" + " services_enabled:\n" + " - legacy\n"; + + Y_ENSURE(Valid(v.Validate(yaml))); + } + + Y_UNIT_TEST(MIRROR_3_DC_NODES) { + auto v = StaticConfigBuilder().CreateValidator(); + const char* yaml = + "static_erasure: mirror-3-dc\n" + "host_configs:\n" + "- drive:\n" + " - path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " type: SSD\n" + " - path: /dev/disk/by-partlabel/ydb_disk_ssd_02\n" + " type: SSD\n" + " - path: /dev/disk/by-partlabel/ydb_disk_ssd_03\n" + " type: SSD\n" + " host_config_id: 1\n" + "hosts:\n" + "- host: ydb-node-zone-a.local\n" + " host_config_id: 1 # numeric host configuration template identifier\n" + " walle_location: # this parameter describes where host is located.\n" + " body: 1 # string representing a host serial number.\n" + " data_center: 'zone-a' # string representing the datacenter / availability zone where the host is located.\n" + " # if cluster is deployed using mirror-3-dc fault tolerance mode, all hosts must be distributed\n" + " # across 3 datacenters.\n" + " rack: '1' # string representing a rack identifier where the host is located.\n" + " # if cluster is deployed using block-4-2 erasure, all hosts should be distrubited\n" + " # accross at least 8 racks.\n" + "- host: ydb-node-zone-b.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 2\n" + " data_center: 'zone-b'\n" + " rack: '2'\n" + "- host: ydb-node-zone-c.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 3\n" + " data_center: 'zone-c'\n" + " rack: '3'\n" + "domains_config:\n" + " # There can be only one root domain in a cluster. Domain name prefixes all scheme objects names, e.g. full name of a table table1 in database db1.\n" + " # in a cluster with domains_config.domain.name parameter set to Root would be equal to /Root/db1/table1\n" + " domain:\n" + " - name: Root\n" + " storage_pool_types:\n" + " - kind: ssd\n" + " pool_config:\n" + " box_id: 1\n" + " # fault tolerance mode name - none, block-4-2, or mirror-3-dc..\n" + " # See docs for more details https://ydb.tech/en/docs/deploy/configuration/config#domains-blob\n" + " erasure_species: mirror-3-dc\n" + " kind: ssd\n" + " geometry:\n" + " realm_level_begin: 10\n" + " realm_level_end: 20\n" + " domain_level_begin: 10\n" + " domain_level_end: 256\n" + " pdisk_filter:\n" + " - property:\n" + " - type: SSD # device type to match host_configs.drive.type\n" + " vdisk_kind: Default\n" + " state_storage:\n" + " - ring:\n" + " node: [1, 2, 3]\n" + " nto_select: 3\n" + " ssid: 1\n" + "table_service_config:\n" + " sql_version: 1\n" + "actor_system_config: # the configuration of the actor system which descibes how cores of the instance are distributed\n" + " executor: # accross different types of workloads in the instance.\n" + " - name: System # system executor of the actor system. in this executor YDB launches system type of workloads, like system tablets\n" + " # and reads from storage.\n" + " threads: 2 # the number of threads allocated to system executor.\n" + " type: BASIC\n" + " - name: User # user executor of the actor system. In this executor YDB launches user workloads, like datashard activities,\n" + " # queries and rpc calls.\n" + " threads: 3 # the number of threads allocated to user executor.\n" + " type: BASIC\n" + " - name: Batch # user executor of the actor system. In this executor YDB launches batch operations, like scan queries, table\n" + " # compactions, background compactions.\n" + " threads: 2 # the number of threads allocated to the batch executor.\n" + " type: BASIC\n" + " - name: IO # the io executor. In this executor launches sync operations and writes logs.\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: IO\n" + " - name: IC # the interconnect executor which YDB uses for network communications accross different nodes of the cluster.\n" + " spin_threshold: 10\n" + " threads: 1 # the number of threads allocated to the interconnect executor.\n" + " time_per_mailbox_micro_secs: 100\n" + " type: BASIC\n" + " scheduler:\n" + " progress_threshold: 10000\n" + " resolution: 256\n" + " spin_threshold: 0\n" + "blob_storage_config: # configuration of static blobstorage group.\n" + " # YDB uses this group to store system tablets' data, like SchemeShard\n" + " service_set:\n" + " groups:\n" + " - erasure_species: mirror-3-dc # fault tolerance mode name for the static group\n" + " rings: # in mirror-3-dc must have exactly 3 rings or availability zones\n" + " - fail_domains: # first record: fail domains of the static group describe where each vdisk of the static group should be located.\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-a.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-a.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_02\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-a.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_03\n" + " - fail_domains: # second ring: fail domains of the static group describe where each vdisk of the static group should be located.\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-b.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-b.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_02\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-b.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_03\n" + " - fail_domains: # third ring: fail domains of the static group describe where each vdisk of the static group should be located.\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-c.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-c.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_02\n" + " - vdisk_locations:\n" + " - node_id: ydb-node-zone-c.local\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_03\n" + "channel_profile_config:\n" + " profile:\n" + " - channel:\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 0\n" + " storage_pool_kind: ssd\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 0\n" + " storage_pool_kind: ssd\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 0\n" + " storage_pool_kind: ssd\n" + " profile_id: 0\n" + "interconnect_config:\n" + " start_tcp: true\n" + " encryption_mode: OPTIONAL\n" + " path_to_certificate_file: \"/opt/ydb/certs/node.crt\"\n" + " path_to_private_key_file: \"/opt/ydb/certs/node.key\"\n" + " path_to_ca_file: \"/opt/ydb/certs/ca.crt\"\n" + "grpc_config:\n" + " cert: \"/opt/ydb/certs/node.crt\"\n" + " key: \"/opt/ydb/certs/node.key\"\n" + " ca: \"/opt/ydb/certs/ca.crt\"\n" + " services_enabled:\n" + " - legacy\n"; + + Y_ENSURE(Valid(v.Validate(yaml))); + } + + Y_UNIT_TEST(MIRROR_3_DC_NODES_IN_MEMORY) { + auto v = StaticConfigBuilder().CreateValidator(); + const char* yaml = + "static_erasure: mirror-3-dc\n" + "host_configs:\n" + "- drive:\n" + " - path: SectorMap:1:64\n" + " type: SSD\n" + " - path: SectorMap:2:64\n" + " type: SSD\n" + " - path: SectorMap:3:64\n" + " type: SSD\n" + " host_config_id: 1\n" + "hosts:\n" + "- host: localhost\n" + " host_config_id: 1\n" + " port: 19001\n" + " walle_location:\n" + " body: 1\n" + " data_center: '1'\n" + " rack: '1'\n" + "- host: localhost\n" + " host_config_id: 1\n" + " port: 19002\n" + " walle_location:\n" + " body: 2\n" + " data_center: '2'\n" + " rack: '2'\n" + "- host: localhost\n" + " host_config_id: 1\n" + " port: 19003\n" + " walle_location:\n" + " body: 3\n" + " data_center: '3'\n" + " rack: '3'\n" + "domains_config:\n" + " domain:\n" + " - name: Root\n" + " storage_pool_types:\n" + " - kind: ssd\n" + " pool_config:\n" + " box_id: 1\n" + " erasure_species: mirror-3-dc\n" + " kind: ssd\n" + " geometry:\n" + " realm_level_begin: 10\n" + " realm_level_end: 20\n" + " domain_level_begin: 10\n" + " domain_level_end: 256\n" + " pdisk_filter:\n" + " - property:\n" + " - type: SSD\n" + " vdisk_kind: Default\n" + " state_storage:\n" + " - ring:\n" + " node: [1, 2, 3]\n" + " nto_select: 3\n" + " ssid: 1\n" + "table_service_config:\n" + " sql_version: 1\n" + "actor_system_config:\n" + " executor:\n" + " - name: System\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: User\n" + " spin_threshold: 0\n" + " threads: 3\n" + " type: BASIC\n" + " - name: Batch\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: IO\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: IO\n" + " - name: IC\n" + " spin_threshold: 10\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: BASIC\n" + " scheduler:\n" + " progress_threshold: 10000\n" + " resolution: 256\n" + " spin_threshold: 0\n" + "blob_storage_config:\n" + " service_set:\n" + " groups:\n" + " - erasure_species: mirror-3-dc\n" + " rings:\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19001\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:1:64\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19001\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:2:64\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19001\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:3:64\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19002\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:1:64\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19002\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:2:64\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19002\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:3:64\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19003\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:1:64\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19003\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:2:64\n" + " - vdisk_locations:\n" + " - node_id: \"localhost:19003\"\n" + " pdisk_category: SSD\n" + " path: SectorMap:3:64\n" + "channel_profile_config:\n" + " profile:\n" + " - channel:\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " profile_id: 0\n"; + + Y_ENSURE(Valid(v.Validate(yaml))); + } + + Y_UNIT_TEST(MIRROR_3_DC_9_NODES) { + auto v = StaticConfigBuilder().CreateValidator(); + const char* yaml = + "static_erasure: mirror-3-dc\n" + "host_configs:\n" + "- drive:\n" + " - path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " type: SSD\n" + " - path: /dev/disk/by-partlabel/ydb_disk_ssd_02\n" + " type: SSD\n" + " host_config_id: 1\n" + "hosts:\n" + "- host: ydb-node-zone-a-1.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 1\n" + " data_center: 'zone-a'\n" + " rack: '1'\n" + "- host: ydb-node-zone-a-2.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 2\n" + " data_center: 'zone-a'\n" + " rack: '2'\n" + "- host: ydb-node-zone-a-3.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 3\n" + " data_center: 'zone-a'\n" + " rack: '3'\n" + "\n" + "- host: ydb-node-zone-b-1.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 4\n" + " data_center: 'zone-b'\n" + " rack: '4'\n" + "- host: ydb-node-zone-b-2.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 5\n" + " data_center: 'zone-b'\n" + " rack: '5'\n" + "- host: ydb-node-zone-b-3.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 6\n" + " data_center: 'zone-b'\n" + " rack: '6'\n" + "\n" + "- host: ydb-node-zone-c-1.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 7\n" + " data_center: 'zone-c'\n" + " rack: '7'\n" + "- host: ydb-node-zone-c-2.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 8\n" + " data_center: 'zone-c'\n" + " rack: '8'\n" + "- host: ydb-node-zone-c-3.local\n" + " host_config_id: 1\n" + " walle_location:\n" + " body: 9\n" + " data_center: 'zone-c'\n" + " rack: '9'\n" + "\n" + "domains_config:\n" + " domain:\n" + " - name: Root\n" + " storage_pool_types:\n" + " - kind: ssd\n" + " pool_config:\n" + " box_id: 1\n" + " erasure_species: mirror-3-dc\n" + " kind: ssd\n" + " pdisk_filter:\n" + " - property:\n" + " - type: SSD\n" + " vdisk_kind: Default\n" + " state_storage:\n" + " - ring:\n" + " node: [1, 2, 3, 4, 5, 6, 7, 8, 9]\n" + " nto_select: 9\n" + " ssid: 1\n" + "table_service_config:\n" + " sql_version: 1\n" + "actor_system_config:\n" + " executor:\n" + " - name: System\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: User\n" + " spin_threshold: 0\n" + " threads: 3\n" + " type: BASIC\n" + " - name: Batch\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: IO\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: IO\n" + " - name: IC\n" + " spin_threshold: 10\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: BASIC\n" + " scheduler:\n" + " progress_threshold: 10000\n" + " resolution: 256\n" + " spin_threshold: 0\n" + "blob_storage_config:\n" + " service_set:\n" + " groups:\n" + " - erasure_species: mirror-3-dc\n" + " rings:\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-1.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-2.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-a-3.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-b-1.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-b-2.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-b-3.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-c-1.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-c-2.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + " - vdisk_locations:\n" + " - node_id: \"ydb-node-zone-c-3.local\"\n" + " pdisk_category: SSD\n" + " path: /dev/disk/by-partlabel/ydb_disk_ssd_01\n" + "channel_profile_config:\n" + " profile:\n" + " - channel:\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: mirror-3-dc\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " profile_id: 0\n" + "interconnect_config:\n" + " start_tcp: true\n" + " encryption_mode: OPTIONAL\n" + " path_to_certificate_file: \"/opt/ydb/certs/node.crt\"\n" + " path_to_private_key_file: \"/opt/ydb/certs/node.key\"\n" + " path_to_ca_file: \"/opt/ydb/certs/ca.crt\"\n" + "grpc_config:\n" + " cert: \"/opt/ydb/certs/node.crt\"\n" + " key: \"/opt/ydb/certs/node.key\"\n" + " ca: \"/opt/ydb/certs/ca.crt\"\n" + " services_enabled:\n" + " - legacy\n"; + + Y_ENSURE(Valid(v.Validate(yaml))); + } + + Y_UNIT_TEST(SINGLE_NODE_IN_MEMORY) { + auto v = StaticConfigBuilder().CreateValidator(); + const char* yaml = + "static_erasure: none\n" + "host_configs:\n" + "- drive:\n" + " - path: SectorMap:1:64\n" + " type: SSD\n" + " host_config_id: 1\n" + "hosts:\n" + "- host: localhost\n" + " host_config_id: 1\n" + " port: 19001\n" + " walle_location:\n" + " body: 1\n" + " data_center: '1'\n" + " rack: '1'\n" + "domains_config:\n" + " domain:\n" + " - name: Root\n" + " storage_pool_types:\n" + " - kind: ssd\n" + " pool_config:\n" + " box_id: 1\n" + " erasure_species: none\n" + " kind: ssd\n" + " pdisk_filter:\n" + " - property:\n" + " - type: SSD\n" + " vdisk_kind: Default\n" + " state_storage:\n" + " - ring:\n" + " node:\n" + " - 1\n" + " nto_select: 1\n" + " ssid: 1\n" + "table_service_config:\n" + " sql_version: 1\n" + "actor_system_config:\n" + " executor:\n" + " - name: System\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: User\n" + " spin_threshold: 0\n" + " threads: 3\n" + " type: BASIC\n" + " - name: Batch\n" + " spin_threshold: 0\n" + " threads: 2\n" + " type: BASIC\n" + " - name: IO\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: IO\n" + " - name: IC\n" + " spin_threshold: 10\n" + " threads: 1\n" + " time_per_mailbox_micro_secs: 100\n" + " type: BASIC\n" + " scheduler:\n" + " progress_threshold: 10000\n" + " resolution: 256\n" + " spin_threshold: 0\n" + "blob_storage_config:\n" + " service_set:\n" + " groups:\n" + " - erasure_species: none\n" + " rings:\n" + " - fail_domains:\n" + " - vdisk_locations:\n" + " - node_id: 1\n" + " path: SectorMap:1:64\n" + " pdisk_category: SSD\n" + "channel_profile_config:\n" + " profile:\n" + " - channel:\n" + " - erasure_species: none\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: none\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " - erasure_species: none\n" + " pdisk_category: 1\n" + " storage_pool_kind: ssd\n" + " profile_id: 0\n"; + + Y_ENSURE(Valid(v.Validate(yaml))); + } + +} diff --git a/ydb/library/yaml_config/static_validator/ut/example_configs/ya.make b/ydb/library/yaml_config/static_validator/ut/example_configs/ya.make new file mode 100644 index 0000000000..edb78a7fdd --- /dev/null +++ b/ydb/library/yaml_config/static_validator/ut/example_configs/ya.make @@ -0,0 +1,12 @@ +UNITTEST() + +SRCS( + test.cpp +) + +PEERDIR( + ydb/library/yaml_config/static_validator + ydb/library/yaml_config/validator +) + +END() diff --git a/ydb/library/yaml_config/static_validator/ut/test.cpp b/ydb/library/yaml_config/static_validator/ut/test.cpp index 555634e1e9..f635c4d770 100644 --- a/ydb/library/yaml_config/static_validator/ut/test.cpp +++ b/ydb/library/yaml_config/static_validator/ut/test.cpp @@ -186,7 +186,7 @@ Y_UNIT_TEST_SUITE(StaticValidator) { " rack: '1'\n"; Y_ENSURE(HasOnlyThisIssues(v.Validate(yaml), { - {"/hosts", "Check \"Must not have to hosts with same host name and port\" failed: items with indexes 0 and 1 are conflicting"} + {"/hosts", "Check \"Must not have two hosts with same host name and port\" failed: items with indexes 0 and 1 are conflicting"} })); yaml = @@ -210,7 +210,7 @@ Y_UNIT_TEST_SUITE(StaticValidator) { Y_ENSURE(HasOnlyThisIssues(v.Validate(yaml), { {"/hosts", "Check \"All array items, that located in \"node_id\" must be unique\" failed: items with indexes 0 and 1 are conflicting"}, - {"/hosts", "Check \"Must not have to hosts with same host name and port\" failed: items with indexes 0 and 1 are conflicting"} + {"/hosts", "Check \"Must not have two hosts with same host name and port\" failed: items with indexes 0 and 1 are conflicting"} })); yaml = @@ -259,8 +259,8 @@ Y_UNIT_TEST_SUITE(StaticValidator) { " vdisk_kind: Default\n" " state_storage:\n" " - ring:\n" - " node: [1, 2, 3, 4, 5, 6, 7, 8]\n" - " nto_select: 8\n" + " node: [1, 2, 3, 4, 5, 6, 7, 8, 9]\n" + " nto_select: 9\n" " ssid: 1\n" " security_config:\n" " enforce_user_token_requirement: true\n"; @@ -283,8 +283,8 @@ Y_UNIT_TEST_SUITE(StaticValidator) { " vdisk_kind: Default\n" " state_storage:\n" " - ring:\n" - " node: [1, 2, 3, 4, 5, 6, 7, 8]\n" - " nto_select: 8\n" + " node: [1, 2, 3, 4, 5, 6, 7, 8, 9]\n" + " nto_select: 9\n" " ssid: 1\n"; Y_ENSURE(HasOnlyThisIssues(v.Validate(yaml), { diff --git a/ydb/library/yaml_config/static_validator/ut/ya.make b/ydb/library/yaml_config/static_validator/ut/ya.make index edb78a7fdd..025a58f980 100644 --- a/ydb/library/yaml_config/static_validator/ut/ya.make +++ b/ydb/library/yaml_config/static_validator/ut/ya.make @@ -10,3 +10,5 @@ PEERDIR( ) END() + +RECURSE_FOR_TESTS(example_configs) diff --git a/ydb/library/yaml_config/validator/configurators.cpp b/ydb/library/yaml_config/validator/configurators.cpp index cd9529f928..adfaecfdb0 100644 --- a/ydb/library/yaml_config/validator/configurators.cpp +++ b/ydb/library/yaml_config/validator/configurators.cpp @@ -28,6 +28,9 @@ std::function<void(TArrayBuilder&)> uniqueByPath(TString path, TString checkName for (int i = 0; i < array.Length(); ++i) { TNodeWrapper node = array[i]; node = walkFrom(node, pathTokens); + if (!node.Exists()) { + continue; + } if (!node.IsScalar()) { ythrow yexception() << "Can't check uniqness of non-scalar fields"; } @@ -62,16 +65,20 @@ TNodeWrapper walkFrom(TNodeWrapper node, TString path) { TNodeWrapper walkFrom(TNodeWrapper node, const TVector<TString>& pathTokens) { for (const TString& pathToken : pathTokens) { - if (node.IsMap()) { - node = node.Map()[pathToken]; - } else if (node.IsArray()) { - auto i = TryFromString<size_t>(pathToken); - if (!i.Defined()) { - ythrow yexception() << "incorrect array index"; + if (node.Exists()) { + if (node.IsMap()) { + node = node.Map()[pathToken]; + } else if (node.IsArray()) { + auto i = TryFromString<size_t>(pathToken); + if (!i.Defined()) { + ythrow yexception() << "incorrect array index"; + } + node = node.Array()[i.GetRef()]; + } else { + ythrow yexception() << "incorrect path"; } - node = node.Array()[i.GetRef()]; } else { - ythrow yexception() << "incorrect path"; + node = node.Map()[pathToken]; } } return node; diff --git a/ydb/library/yaml_config/validator/validator_builder.cpp b/ydb/library/yaml_config/validator/validator_builder.cpp index 8826950204..86b54c4bea 100644 --- a/ydb/library/yaml_config/validator/validator_builder.cpp +++ b/ydb/library/yaml_config/validator/validator_builder.cpp @@ -513,13 +513,23 @@ TBoolValidator TBoolBuilder::CreateValidator() { // TEnumBuilder TEnumBuilder::TEnumBuilder(THashSet<TString> items) - : TBase(ENodeType::Enum), Items_(std::move(items)) {} + : TBase(ENodeType::Enum) { + for (TString item : items) { + item.to_lower(); + Items_.insert(item); + } + } TEnumBuilder::TEnumBuilder(const TEnumBuilder& builder) : TBase(builder), Items_(builder.Items_) {} TEnumBuilder::TEnumBuilder(TEnumBuilder&& builder) - : TBase(std::move(builder)), Items_(std::move(builder.Items_)) {} + : TBase(std::move(builder)), Items_(std::move(builder.Items_)) { + for (TString item : builder.Items_) { + item.to_lower(); + Items_.insert(item); + } + } TEnumBuilder& TEnumBuilder::operator=(const TEnumBuilder& builder) { TBuilder::operator=(builder); @@ -538,6 +548,15 @@ TEnumBuilder::TEnumBuilder(std::function<void(TEnumBuilder&)> configurator) configurator(*this); } +TEnumBuilder& TEnumBuilder::SetItems(THashSet<TString> items) { + Items_.clear(); + for (TString item : items) { + item.to_lower(); + Items_.insert(item); + } + return *this; +} + TEnumValidator TEnumBuilder::CreateValidator() { auto result = TEnumValidator(); result.Checkers_ = Checkers_; diff --git a/ydb/library/yaml_config/validator/validator_builder.h b/ydb/library/yaml_config/validator/validator_builder.h index 0cee26aa2a..4096ca7e85 100644 --- a/ydb/library/yaml_config/validator/validator_builder.h +++ b/ydb/library/yaml_config/validator/validator_builder.h @@ -261,7 +261,7 @@ public: TEnumBuilder(std::function<void(TEnumBuilder&)> configurator); - void SetItems(const THashSet<TString>& items); + TEnumBuilder& SetItems(THashSet<TString> items); TEnumValidator CreateValidator(); |