diff options
author | vturov <vturov@yandex-team.com> | 2024-07-10 11:36:47 +0300 |
---|---|---|
committer | vturov <vturov@yandex-team.com> | 2024-07-10 11:52:05 +0300 |
commit | 4665fe89c77e887284145072ac2cadefada81305 (patch) | |
tree | 4ad82789f9325e3833e1654397748be2cf0c6d45 /build/plugins/nots.py | |
parent | 656c77fab903266e4dc0dc64fdf7f7d759e249f3 (diff) | |
download | ydb-4665fe89c77e887284145072ac2cadefada81305.tar.gz |
откатываем изменения rXXXXXX
Откат изменений, произведённых в rXXXXXX до подробного разбора необходимости этих изменений
955fb30f5aa99fde5d346efde4745a1c525a2c32
Diffstat (limited to 'build/plugins/nots.py')
-rw-r--r-- | build/plugins/nots.py | 488 |
1 files changed, 202 insertions, 286 deletions
diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 7fd69d2389..4a2f8474da 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -1,13 +1,10 @@ +import base64 +import six import os - -import _dart_fields as df import ymake import ytest -from _dart_fields import ( - create_dart_record, - _create_pm, -) -from _common import rootrel_arc_src, to_yesno + +from _common import resolve_common_const, get_norm_unit_path, rootrel_arc_src, to_yesno # 1 is 60 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL) @@ -16,23 +13,6 @@ from _common import rootrel_arc_src, to_yesno ESLINT_FILE_PROCESSING_TIME_DEFAULT = 0.2 # seconds per file -TS_TEST_FIELDS_BASE = ( - df.BinaryPath.value4, - df.BuildFolderPath.value, - df.ForkMode.value2, - df.NodejsRootVarName.value, - df.ScriptRelPath.value2, - df.SourceFolderPath.value, - df.SplitFactor.value2, - df.TestData.value7, - df.TestedProjectName.value7, - df.TestEnv.value, - df.TestName.value, - df.TestRecipes.value, - df.TestTimeout.value3, -) - - class PluginLogger(object): def __init__(self): self.unit = None @@ -77,6 +57,30 @@ class PluginLogger(object): logger = PluginLogger() +def get_values_list(unit, key): + res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split()) + return [r for r in res if r and r not in ['""', "''"]] + + +def format_recipes(data: str | None) -> str: + if not data: + return "" + + data = data.replace('"USE_RECIPE_DELIM"', "\n") + data = data.replace("$TEST_RECIPES_VALUE", "") + return data + + +def prepare_recipes(data: str | None) -> bytes: + formatted = format_recipes(data) + return base64.b64encode(six.ensure_binary(formatted)) + + +def serialize_list(lst): + lst = list(filter(None, lst)) + return '\"' + ';'.join(lst) + '\"' if lst else '' + + def _with_report_configure_error(fn): def _wrapper(*args, **kwargs): last_state = logger.get_state() @@ -114,6 +118,26 @@ def _build_cmd_input_paths(paths, hide=False, disable_include_processor=False): return _build_directives("input", [hide_part, disable_ip_part], paths) +def _create_pm(unit): + from lib.nots.package_manager import manager + + sources_path = unit.path() + module_path = unit.get("MODDIR") + if unit.get("TS_TEST_FOR"): + sources_path = unit.get("TS_TEST_FOR_DIR") + module_path = unit.get("TS_TEST_FOR_PATH") + + return manager( + sources_path=unit.resolve(sources_path), + build_root="$B", + build_path=unit.path().replace("$S", "$B", 1), + contribs_path=unit.get("NPM_CONTRIBS_PATH"), + nodejs_bin_path=None, + script_path=None, + module_path=module_path, + ) + + def _create_erm_json(unit): from lib.nots.erm_json_lite import ErmJsonLite @@ -254,7 +278,7 @@ def on_ts_configure(unit): _filter_inputs_by_rules_from_tsconfig(unit, tsconfig) _setup_eslint(unit) - _setup_tsc_typecheck(unit) + _setup_tsc_typecheck(unit, tsconfig_paths) if unit.get("TS_YNDEXING") == "yes": unit.on_do_ts_yndexing() @@ -314,279 +338,112 @@ def _filter_inputs_by_rules_from_tsconfig(unit, tsconfig): __set_append(unit, "TS_INPUT_FILES", [os.path.join(target_path, f) for f in filtered_files]) -def _is_tests_enabled(unit): - if unit.get("TIDY") == "yes": - return False - - return True - - -@df.with_fields( - TS_TEST_FIELDS_BASE - + ( - df.Size.value2, - df.Tag.value2, - df.Requirements.value4, - df.ConfigPath.value, - df.TsTestDataDirs.value, - df.TsTestDataDirsRename.value, - df.TsResources.value, +def _get_ts_test_data_dirs(unit): + return sorted( + set([os.path.dirname(rootrel_arc_src(p, unit)) for p in (get_values_list(unit, "_TS_TEST_DATA_VALUE") or [])]) ) -) -def _add_jest_ts_test(fields, unit, default_config, node_modules_filename): - if unit.enabled('TS_COVERAGE'): - unit.on_peerdir_ts_resource("nyc") - - for_mod_path = df.TsTestForPath.value(unit, (), {})[df.TsTestForPath.KEY] - - # for_mod_path = unit.get("TS_TEST_FOR_PATH") - unit.onpeerdir([for_mod_path]) - unit.on_setup_extract_node_modules_recipe([for_mod_path]) - unit.on_setup_extract_output_tars_recipe([for_mod_path]) - - test_runner = 'jest' - - unit.set(["TS_TEST_NM", os.path.join("$(BUILD_ROOT)", for_mod_path, node_modules_filename)]) - - config_path = unit.get("TS_TEST_CONFIG_PATH") - if not config_path: - config_path = os.path.join(for_mod_path, default_config) - unit.set(["TS_TEST_CONFIG_PATH", config_path]) - - test_files = df.TestFiles.value6(unit, (), {})[df.TestFiles.KEY] - if not test_files: - ymake.report_configure_error("No tests found") - return - from lib.nots.package_manager import constants - def sort_uniq(text): - return sorted(set(text)) +def _resolve_config_path(unit, test_runner, rel_to): + config_path = unit.get("ESLINT_CONFIG_PATH") if test_runner == "eslint" else unit.get("TS_TEST_CONFIG_PATH") + arc_config_path = unit.resolve_arc_path(config_path) + abs_config_path = unit.resolve(arc_config_path) + if not abs_config_path: + raise Exception("{} config not found: {}".format(test_runner, config_path)) - deps = df.CustomDependencies.value5(unit, (), {})[df.CustomDependencies.KEY].split() + unit.onsrcs([arc_config_path]) + abs_rel_to = unit.resolve(unit.resolve_arc_path(unit.get(rel_to))) + return os.path.relpath(abs_config_path, start=abs_rel_to) - if deps: - joined_deps = "\n".join(deps) - logger.info(f"{test_runner} deps: \n{joined_deps}") - unit.ondepends(deps) - flat_args = (test_runner, "TS_TEST_FOR_PATH") +def _is_tests_enabled(unit): + if unit.get("TIDY") == "yes": + return False - dart_record = create_dart_record(fields, unit, flat_args, {}) - dart_record[df.TestFiles.KEY] = test_files - dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME + return True - extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() - dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) - dart_record[df.TsTestForPath.KEY] = for_mod_path - data = ytest.dump_test(unit, dart_record) - if data: - unit.set_property(["DART_DATA", data]) +def _get_test_runner_handlers(): + return { + "jest": _add_jest_ts_test, + "hermione": _add_hermione_ts_test, + "playwright": _add_playwright_ts_test, + } -@df.with_fields( - TS_TEST_FIELDS_BASE - + ( - df.Tag.value3, - df.Requirements.value5, - df.ConfigPath.value, - df.TsTestDataDirs.value, - df.TsTestDataDirsRename.value, - df.TsResources.value, +def _add_jest_ts_test(unit, test_runner, test_files, deps, test_record): + test_record.update( + { + "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"), + } ) -) -def _add_hermione_ts_test(fields, unit, default_config, node_modules_filename): - if unit.enabled('TS_COVERAGE'): - unit.on_peerdir_ts_resource("nyc") - - for_mod_path = df.TsTestForPath.value(unit, (), {})[df.TsTestForPath.KEY] - # for_mod_path = unit.get("TS_TEST_FOR_PATH") - unit.onpeerdir([for_mod_path]) - unit.on_setup_extract_node_modules_recipe([for_mod_path]) - unit.on_setup_extract_output_tars_recipe([for_mod_path]) + _add_test(unit, test_runner, test_files, deps, test_record) - test_runner = 'hermione' - unit.set(["TS_TEST_NM", os.path.join("$B", for_mod_path, node_modules_filename)]) +def _add_hermione_ts_test(unit, test_runner, test_files, deps, test_record): + test_tags = sorted(set(["ya:fat", "ya:external", "ya:noretries"] + get_values_list(unit, "TEST_TAGS_VALUE"))) + test_requirements = sorted(set(["network:full"] + get_values_list(unit, "TEST_REQUIREMENTS_VALUE"))) - config_path = unit.get("TS_TEST_CONFIG_PATH") - if not config_path: - config_path = os.path.join(for_mod_path, default_config) - unit.set(["TS_TEST_CONFIG_PATH", config_path]) - - test_files = df.TestFiles.value6(unit, (), {})[df.TestFiles.KEY] - if not test_files: - ymake.report_configure_error("No tests found") - return - - from lib.nots.package_manager import constants - - def sort_uniq(text): - return sorted(set(text)) - - deps = df.CustomDependencies.value5(unit, (), {})[df.CustomDependencies.KEY].split() - - if deps: - joined_deps = "\n".join(deps) - logger.info(f"{test_runner} deps: \n{joined_deps}") - unit.ondepends(deps) - - flat_args = (test_runner, "TS_TEST_FOR_PATH") - - dart_record = create_dart_record(fields, unit, flat_args, {}) - dart_record[df.TestFiles.KEY] = test_files - dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME - - extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() - dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) - dart_record[df.TsTestForPath.KEY] = for_mod_path - dart_record[df.Size.KEY] = "LARGE" - - data = ytest.dump_test(unit, dart_record) - if data: - unit.set_property(["DART_DATA", data]) - - -@df.with_fields( - TS_TEST_FIELDS_BASE - + ( - df.Size.value2, - df.Tag.value2, - df.Requirements.value4, - df.ConfigPath.value, - df.TsTestDataDirs.value, - df.TsTestDataDirsRename.value, - df.TsResources.value, + test_record.update( + { + "SIZE": "LARGE", + "TAG": serialize_list(test_tags), + "REQUIREMENTS": serialize_list(test_requirements), + "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"), + } ) -) -def _add_playwright_ts_test(fields, unit, default_config, node_modules_filename): - if unit.enabled('TS_COVERAGE'): - unit.on_peerdir_ts_resource("nyc") - - for_mod_path = unit.get("TS_TEST_FOR_PATH") - unit.onpeerdir([for_mod_path]) - unit.on_setup_extract_node_modules_recipe([for_mod_path]) - unit.on_setup_extract_output_tars_recipe([for_mod_path]) - test_runner = 'playwright' + _add_test(unit, test_runner, test_files, deps, test_record) - unit.set(["TS_TEST_NM", os.path.join("$(BUILD_ROOT)", for_mod_path, node_modules_filename)]) - config_path = unit.get("TS_TEST_CONFIG_PATH") - if not config_path: - config_path = os.path.join(for_mod_path, default_config) - unit.set(["TS_TEST_CONFIG_PATH", config_path]) - - test_files = df.TestFiles.value6(unit, (), {})[df.TestFiles.KEY] - if not test_files: - ymake.report_configure_error("No tests found") - return - - from lib.nots.package_manager import constants - - def sort_uniq(text): - return sorted(set(text)) - - deps = df.CustomDependencies.value5(unit, (), {})[df.CustomDependencies.KEY].split() - - if deps: - joined_deps = "\n".join(deps) - logger.info(f"{test_runner} deps: \n{joined_deps}") - unit.ondepends(deps) - - flat_args = (test_runner, "TS_TEST_FOR_PATH") - - dart_record = create_dart_record(fields, unit, flat_args, {}) - dart_record[df.TestFiles.KEY] = test_files - dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME - - extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() - dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) - dart_record[df.TsTestForPath.KEY] = for_mod_path - - data = ytest.dump_test(unit, dart_record) - if data: - unit.set_property(["DART_DATA", data]) +def _add_playwright_ts_test(unit, test_runner, test_files, deps, test_record): + test_record.update( + { + "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"), + } + ) + _add_test(unit, test_runner, test_files, deps, test_record) -@df.with_fields( - TS_TEST_FIELDS_BASE - + ( - df.Size.value2, - df.TestCwd.value3, - df.Tag.value2, - df.Requirements.value4, - df.EslintConfigPath.value, - ) -) -def _setup_eslint(fields, unit): +def _setup_eslint(unit): if not _is_tests_enabled(unit): return if unit.get("_NO_LINT_VALUE") == "none": return - test_files = df.TestFiles.value6(unit, (), {})[df.TestFiles.KEY] - if not test_files: + lint_files = get_values_list(unit, "_TS_LINT_SRCS_VALUE") + if not lint_files: return + mod_dir = unit.get("MODDIR") + unit.on_peerdir_ts_resource("eslint") user_recipes = unit.get("TEST_RECIPES_VALUE") unit.on_setup_install_node_modules_recipe() - test_type = "eslint" + lint_files = _resolve_module_files(unit, mod_dir, lint_files) + deps = _create_pm(unit).get_peers_from_package_json() + test_record = { + "ESLINT_CONFIG_PATH": _resolve_config_path(unit, "eslint", rel_to="MODDIR"), + "LINT-FILE-PROCESSING-TIME": str(ESLINT_FILE_PROCESSING_TIME_DEFAULT), + } - from lib.nots.package_manager import constants - - def sort_uniq(text): - return sorted(set(text)) - - deps = df.CustomDependencies.value5(unit, (), {})[df.CustomDependencies.KEY].split() - - if deps: - joined_deps = "\n".join(deps) - logger.info(f"{test_type} deps: \n{joined_deps}") - unit.ondepends(deps) - - flat_args = (test_type, "TS_TEST_FOR_PATH") - - dart_record = create_dart_record(fields, unit, flat_args, {}) - dart_record[df.TestFiles.KEY] = test_files - dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME - - extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() - dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) - dart_record[df.LintFileProcessingTime.KEY] = str(ESLINT_FILE_PROCESSING_TIME_DEFAULT) - - data = ytest.dump_test(unit, dart_record) - if data: - unit.set_property(["DART_DATA", data]) + _add_test(unit, "eslint", lint_files, deps, test_record, mod_dir) unit.set(["TEST_RECIPES_VALUE", user_recipes]) -@df.with_fields( - TS_TEST_FIELDS_BASE - + ( - df.Size.value2, - df.TestCwd.value3, - df.Tag.value2, - df.Requirements.value4, - ) -) -def _setup_tsc_typecheck(fields, unit): +def _setup_tsc_typecheck(unit, tsconfig_paths: list[str]): if not _is_tests_enabled(unit): return if unit.get("_TS_TYPECHECK_VALUE") == "none": return - # typecheck_files = get_values_list(unit, "TS_INPUT_FILES") - test_files = df.TestFiles.value6(unit, (), {})[df.TestFiles.KEY] - if not test_files: + typecheck_files = get_values_list(unit, "TS_INPUT_FILES") + if not typecheck_files: return - tsconfig_paths = unit.get("TS_CONFIG_PATH").split() tsconfig_path = tsconfig_paths[0] if len(tsconfig_paths) > 1: @@ -603,33 +460,14 @@ def _setup_tsc_typecheck(fields, unit): unit.on_setup_install_node_modules_recipe() unit.on_setup_extract_output_tars_recipe([unit.get("MODDIR")]) - test_type = "tsc_typecheck" - - from lib.nots.package_manager import constants - - def sort_uniq(text): - return sorted(set(text)) - - deps = df.CustomDependencies.value5(unit, (), {})[df.CustomDependencies.KEY].split() - - if deps: - joined_deps = "\n".join(deps) - logger.info(f"{test_type} deps: \n{joined_deps}") - unit.ondepends(deps) - - flat_args = (test_type,) - - dart_record = create_dart_record(fields, unit, flat_args, {}) - dart_record[df.TestFiles.KEY] = test_files - dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME - - extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() - dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) - dart_record[df.TsConfigPath.KEY] = tsconfig_path - - data = ytest.dump_test(unit, dart_record) - if data: - unit.set_property(["DART_DATA", data]) + _add_test( + unit, + test_type="tsc_typecheck", + test_files=[resolve_common_const(f) for f in typecheck_files], + deps=_create_pm(unit).get_peers_from_package_json(), + test_record={"TS_CONFIG_PATH": tsconfig_path}, + test_cwd=unit.get("MODDIR"), + ) unit.set(["TEST_RECIPES_VALUE", user_recipes]) @@ -646,6 +484,56 @@ def _resolve_module_files(unit, mod_dir, file_paths): return resolved_files +def _add_test(unit, test_type, test_files, deps=None, test_record=None, test_cwd=None): + from lib.nots.package_manager import constants + + def sort_uniq(text): + return sorted(set(text)) + + recipes_lines = format_recipes(unit.get("TEST_RECIPES_VALUE")).strip().splitlines() + if recipes_lines: + deps = deps or [] + deps.extend([os.path.dirname(r.strip().split(" ")[0]) for r in recipes_lines]) + + if deps: + joined_deps = "\n".join(deps) + logger.info(f"{test_type} deps: \n{joined_deps}") + unit.ondepends(deps) + + test_dir = get_norm_unit_path(unit) + full_test_record = { + # Key to discover suite (see devtools/ya/test/explore/__init__.py#gen_suite) + "SCRIPT-REL-PATH": test_type, + # Test name as shown in PR check, should be unique inside one module + "TEST-NAME": test_type.lower().replace(".new", ""), + "TEST-TIMEOUT": unit.get("TEST_TIMEOUT") or "", + "TEST-ENV": ytest.prepare_env(unit.get("TEST_ENV_VALUE")), + "TESTED-PROJECT-NAME": os.path.splitext(unit.filename())[0], + "TEST-RECIPES": prepare_recipes(unit.get("TEST_RECIPES_VALUE")), + "SOURCE-FOLDER-PATH": test_dir, + "BUILD-FOLDER-PATH": test_dir, + "BINARY-PATH": os.path.join(test_dir, unit.filename()), + "SPLIT-FACTOR": unit.get("TEST_SPLIT_FACTOR") or "", + "FORK-MODE": unit.get("TEST_FORK_MODE") or "", + "SIZE": unit.get("TEST_SIZE_NAME") or "", + "TEST-DATA": serialize_list(get_values_list(unit, "TEST_DATA_VALUE")), + "TEST-FILES": serialize_list(test_files), + "TEST-CWD": test_cwd or "", + "TAG": serialize_list(get_values_list(unit, "TEST_TAGS_VALUE")), + "REQUIREMENTS": serialize_list(get_values_list(unit, "TEST_REQUIREMENTS_VALUE")), + "NODEJS-ROOT-VAR-NAME": unit.get("NODEJS-ROOT-VAR-NAME"), + "NODE-MODULES-BUNDLE-FILENAME": constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, + "CUSTOM-DEPENDENCIES": " ".join(sort_uniq((deps or []) + get_values_list(unit, "TEST_DEPENDS_VALUE"))), + } + + if test_record: + full_test_record.update(test_record) + + data = ytest.dump_test(unit, full_test_record) + if data: + unit.set_property(["DART_DATA", data]) + + def _set_resource_vars(unit, erm_json, tool, version, nodejs_major=None): # type: (any, ErmJsonLite, Version, str|None, int|None) -> None @@ -797,12 +685,40 @@ def on_ts_test_for_configure(unit, test_runner, default_config, node_modules_fil if not _is_tests_enabled(unit): return - if test_runner == 'jest': - _add_jest_ts_test(unit, default_config, node_modules_filename) - elif test_runner == 'hermione': - _add_hermione_ts_test(unit, default_config, node_modules_filename) - elif test_runner == 'playwright': - _add_playwright_ts_test(unit, default_config, node_modules_filename) + if unit.enabled('TS_COVERAGE'): + unit.on_peerdir_ts_resource("nyc") + + for_mod_path = unit.get("TS_TEST_FOR_PATH") + unit.onpeerdir([for_mod_path]) + unit.on_setup_extract_node_modules_recipe([for_mod_path]) + unit.on_setup_extract_output_tars_recipe([for_mod_path]) + + root = "$B" if test_runner == "hermione" else "$(BUILD_ROOT)" + unit.set(["TS_TEST_NM", os.path.join(root, for_mod_path, node_modules_filename)]) + + config_path = unit.get("TS_TEST_CONFIG_PATH") + if not config_path: + config_path = os.path.join(for_mod_path, default_config) + unit.set(["TS_TEST_CONFIG_PATH", config_path]) + + test_record = _add_ts_resources_to_test_record( + unit, + { + "TS-TEST-FOR-PATH": for_mod_path, + "TS-TEST-DATA-DIRS": serialize_list(_get_ts_test_data_dirs(unit)), + "TS-TEST-DATA-DIRS-RENAME": unit.get("_TS_TEST_DATA_DIRS_RENAME_VALUE"), + }, + ) + + test_files = get_values_list(unit, "_TS_TEST_SRCS_VALUE") + test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files) + if not test_files: + ymake.report_configure_error("No tests found") + return + + deps = _create_pm(unit).get_peers_from_package_json() + add_ts_test = _get_test_runner_handlers()[test_runner] + add_ts_test(unit, test_runner, test_files, deps, test_record) @_with_report_configure_error |