diff options
author | alevitskii <alevitskii@yandex-team.com> | 2024-07-23 10:45:19 +0300 |
---|---|---|
committer | alevitskii <alevitskii@yandex-team.com> | 2024-07-23 11:02:04 +0300 |
commit | e368fb4c997be1bf6af58fe8f82d572e19e20007 (patch) | |
tree | 512547240fd24ffa7bae0c4b3ee6a050702e6e88 /build | |
parent | 3768e8faf508c366fe31d91e34f1c94ccbf1151b (diff) | |
download | ydb-e368fb4c997be1bf6af58fe8f82d572e19e20007.tar.gz |
Fix nots test plugins
Fix nots test plugins
Фикс после отката в https://a.yandex-team.ru/review/6265741/details
1. В `_setup_tsc_typecheck` теперь `df.TestFiles.value7` вместо `df.TestFiles.value6`
2. В `_setup_eslint` теперь `df.TestFiles.value8` вместо `df.TestFiles.value6` и `flat_args = (test_type, "MODDIR")` вместо `flat_args = (test_type, "TS_TEST_FOR_PATH")`
4b229b102cfd73a6bbc7100b30e26c6ad0bb210a
Diffstat (limited to 'build')
-rw-r--r-- | build/plugins/_common.py | 4 | ||||
-rw-r--r-- | build/plugins/_dart_fields.py | 39 | ||||
-rw-r--r-- | build/plugins/nots.py | 377 |
3 files changed, 194 insertions, 226 deletions
diff --git a/build/plugins/_common.py b/build/plugins/_common.py index cd1d8024a8..a2b19b5b3d 100644 --- a/build/plugins/_common.py +++ b/build/plugins/_common.py @@ -30,6 +30,10 @@ def listid(items): return pathid(str(sorted(items))) +def sort_uniq(items): + return sorted(set(items)) + + def stripext(fname): return fname[: fname.rfind('.')] diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index 54f2427384..e25e20d148 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -219,15 +219,6 @@ def extract_java_system_properties(unit, args): return props, None -def _create_erm_json(unit): - from lib.nots.erm_json_lite import ErmJsonLite - - erm_packages_path = unit.get("ERM_PACKAGES_PATH") - path = unit.resolve(unit.resolve_arc_path(erm_packages_path)) - - return ErmJsonLite.load(path) - - def _resolve_module_files(unit, mod_dir, file_paths): mod_dir_with_sep_len = len(mod_dir) + 1 resolved_files = [] @@ -241,26 +232,6 @@ def _resolve_module_files(unit, mod_dir, file_paths): return resolved_files -def _create_pm(unit): - from lib.nots.package_manager import manager - - sources_path = unit.path() - module_path = unit.get("MODDIR") - if unit.get("TS_TEST_FOR"): - sources_path = unit.get("TS_TEST_FOR_DIR") - module_path = unit.get("TS_TEST_FOR_PATH") - - return manager( - sources_path=unit.resolve(sources_path), - build_root="$B", - build_path=unit.path().replace("$S", "$B", 1), - contribs_path=unit.get("NPM_CONTRIBS_PATH"), - nodejs_bin_path=None, - script_path=None, - module_path=module_path, - ) - - def _resolve_config_path(unit, test_runner, rel_to): config_path = unit.get("ESLINT_CONFIG_PATH") if test_runner == "eslint" else unit.get("TS_TEST_CONFIG_PATH") arc_config_path = unit.resolve_arc_path(config_path) @@ -402,7 +373,7 @@ class CustomDependencies: @classmethod def value5(cls, unit, flat_args, spec_args): - deps = _create_pm(unit).get_peers_from_package_json() + deps = flat_args[0] recipes_lines = format_recipes(unit.get("TEST_RECIPES_VALUE")).strip().splitlines() if recipes_lines: deps = deps or [] @@ -1014,6 +985,12 @@ class TestFiles: test_files = [_common.resolve_common_const(f) for f in typecheck_files] return {cls.KEY: serialize_list(test_files)} + @classmethod + def value8(cls, unit, flat_args, spec_args): + test_files = get_values_list(unit, "_TS_LINT_SRCS_VALUE") + test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files) + return {cls.KEY: serialize_list(test_files)} + class TestEnv: KEY = 'TEST-ENV' @@ -1147,7 +1124,7 @@ class TsResources: @classmethod def value(cls, unit, flat_args, spec_args): - erm_json = _create_erm_json(unit) + erm_json = spec_args['erm_json'] ret = {} for tool in erm_json.list_npm_packages(): tool_resource_label = cls.KEY.format(tool.upper()) diff --git a/build/plugins/nots.py b/build/plugins/nots.py index b99de011d4..facae56747 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -1,10 +1,11 @@ -import base64 -import six import os +from enum import StrEnum, auto + +import _dart_fields as df import ymake import ytest - -from _common import resolve_common_const, get_norm_unit_path, rootrel_arc_src, strip_roots, to_yesno +from _dart_fields import create_dart_record +from _common import rootrel_arc_src, to_yesno, strip_roots, sort_uniq # 1 is 60 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL) @@ -13,6 +14,76 @@ from _common import resolve_common_const, get_norm_unit_path, rootrel_arc_src, s ESLINT_FILE_PROCESSING_TIME_DEFAULT = 0.2 # seconds per file +class TsTestType(StrEnum): + JEST = auto() + HERMIONE = auto() + PLAYWRIGHT = auto() + ESLINT = auto() + TSC_TYPECHECK = auto() + + +TS_TEST_FIELDS_BASE = ( + df.BinaryPath.value4, + df.BuildFolderPath.value, + df.ForkMode.value2, + df.NodejsRootVarName.value, + df.ScriptRelPath.value2, + df.SourceFolderPath.value, + df.SplitFactor.value2, + df.TestData.value7, + df.TestedProjectName.value7, + df.TestEnv.value, + df.TestName.value, + df.TestRecipes.value, + df.TestTimeout.value3, +) + +TS_TEST_SPECIFIC_FIELDS = { + TsTestType.JEST: ( + df.Size.value2, + df.Tag.value2, + df.Requirements.value4, + df.ConfigPath.value, + df.TsTestDataDirs.value, + df.TsTestDataDirsRename.value, + df.TsResources.value, + df.TsTestForPath.value, + ), + TsTestType.HERMIONE: ( + df.Tag.value3, + df.Requirements.value5, + df.ConfigPath.value, + df.TsTestDataDirs.value, + df.TsTestDataDirsRename.value, + df.TsResources.value, + df.TsTestForPath.value, + ), + TsTestType.PLAYWRIGHT: ( + df.Size.value2, + df.Tag.value2, + df.Requirements.value4, + df.ConfigPath.value, + df.TsTestDataDirs.value, + df.TsTestDataDirsRename.value, + df.TsResources.value, + df.TsTestForPath.value, + ), + TsTestType.ESLINT: ( + df.Size.value2, + df.TestCwd.value3, + df.Tag.value2, + df.Requirements.value4, + df.EslintConfigPath.value, + ), + TsTestType.TSC_TYPECHECK: ( + df.Size.value2, + df.TestCwd.value3, + df.Tag.value2, + df.Requirements.value4, + ), +} + + class PluginLogger(object): def __init__(self): self.unit = None @@ -57,30 +128,6 @@ class PluginLogger(object): logger = PluginLogger() -def get_values_list(unit, key): - res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split()) - return [r for r in res if r and r not in ['""', "''"]] - - -def format_recipes(data: str | None) -> str: - if not data: - return "" - - data = data.replace('"USE_RECIPE_DELIM"', "\n") - data = data.replace("$TEST_RECIPES_VALUE", "") - return data - - -def prepare_recipes(data: str | None) -> bytes: - formatted = format_recipes(data) - return base64.b64encode(six.ensure_binary(formatted)) - - -def serialize_list(lst): - lst = list(filter(None, lst)) - return '\"' + ';'.join(lst) + '\"' if lst else '' - - def _with_report_configure_error(fn): def _wrapper(*args, **kwargs): last_state = logger.get_state() @@ -118,6 +165,15 @@ def _build_cmd_input_paths(paths, hide=False, disable_include_processor=False): return _build_directives("input", [hide_part, disable_ip_part], paths) +def _create_erm_json(unit): + from lib.nots.erm_json_lite import ErmJsonLite + + erm_packages_path = unit.get("ERM_PACKAGES_PATH") + path = unit.resolve(unit.resolve_arc_path(erm_packages_path)) + + return ErmJsonLite.load(path) + + def _get_pm_type(unit) -> str: resolved = unit.get("PM_TYPE") if not resolved: @@ -150,15 +206,6 @@ def _create_pm(unit): ) -def _create_erm_json(unit): - from lib.nots.erm_json_lite import ErmJsonLite - - erm_packages_path = unit.get("ERM_PACKAGES_PATH") - path = unit.resolve(unit.resolve_arc_path(erm_packages_path)) - - return ErmJsonLite.load(path) - - @_with_report_configure_error def on_set_package_manager(unit): pm_type = "pnpm" # projects without any lockfile are processed by pnpm @@ -313,7 +360,7 @@ def on_ts_configure(unit): _filter_inputs_by_rules_from_tsconfig(unit, tsconfig) _setup_eslint(unit) - _setup_tsc_typecheck(unit, tsconfig_paths) + _setup_tsc_typecheck(unit) if unit.get("TS_YNDEXING") == "yes": unit.on_do_ts_yndexing() @@ -373,24 +420,6 @@ def _filter_inputs_by_rules_from_tsconfig(unit, tsconfig): __set_append(unit, "TS_INPUT_FILES", [os.path.join(target_path, f) for f in filtered_files]) -def _get_ts_test_data_dirs(unit): - return sorted( - set([os.path.dirname(rootrel_arc_src(p, unit)) for p in (get_values_list(unit, "_TS_TEST_DATA_VALUE") or [])]) - ) - - -def _resolve_config_path(unit, test_runner, rel_to): - config_path = unit.get("ESLINT_CONFIG_PATH") if test_runner == "eslint" else unit.get("TS_TEST_CONFIG_PATH") - arc_config_path = unit.resolve_arc_path(config_path) - abs_config_path = unit.resolve(arc_config_path) - if not abs_config_path: - raise Exception("{} config not found: {}".format(test_runner, config_path)) - - unit.onsrcs([arc_config_path]) - abs_rel_to = unit.resolve(unit.resolve_arc_path(unit.get(rel_to))) - return os.path.relpath(abs_config_path, start=abs_rel_to) - - def _is_tests_enabled(unit): if unit.get("TIDY") == "yes": return False @@ -398,48 +427,6 @@ def _is_tests_enabled(unit): return True -def _get_test_runner_handlers(): - return { - "jest": _add_jest_ts_test, - "hermione": _add_hermione_ts_test, - "playwright": _add_playwright_ts_test, - } - - -def _add_jest_ts_test(unit, test_runner, test_files, deps, test_record): - test_record.update( - { - "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"), - } - ) - _add_test(unit, test_runner, test_files, deps, test_record) - - -def _add_hermione_ts_test(unit, test_runner, test_files, deps, test_record): - test_tags = sorted(set(["ya:fat", "ya:external", "ya:noretries"] + get_values_list(unit, "TEST_TAGS_VALUE"))) - test_requirements = sorted(set(["network:full"] + get_values_list(unit, "TEST_REQUIREMENTS_VALUE"))) - - test_record.update( - { - "SIZE": "LARGE", - "TAG": serialize_list(test_tags), - "REQUIREMENTS": serialize_list(test_requirements), - "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"), - } - ) - - _add_test(unit, test_runner, test_files, deps, test_record) - - -def _add_playwright_ts_test(unit, test_runner, test_files, deps, test_record): - test_record.update( - { - "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"), - } - ) - _add_test(unit, test_runner, test_files, deps, test_record) - - def _setup_eslint(unit): if not _is_tests_enabled(unit): return @@ -447,38 +434,59 @@ def _setup_eslint(unit): if unit.get("_NO_LINT_VALUE") == "none": return - lint_files = get_values_list(unit, "_TS_LINT_SRCS_VALUE") - if not lint_files: + test_files = df.TestFiles.value8(unit, (), {})[df.TestFiles.KEY] + if not test_files: return - mod_dir = unit.get("MODDIR") - unit.on_peerdir_ts_resource("eslint") user_recipes = unit.get("TEST_RECIPES_VALUE") unit.on_setup_install_node_modules_recipe() - lint_files = _resolve_module_files(unit, mod_dir, lint_files) - deps = _create_pm(unit).get_peers_from_package_json() - test_record = { - "ESLINT_CONFIG_PATH": _resolve_config_path(unit, "eslint", rel_to="MODDIR"), - "LINT-FILE-PROCESSING-TIME": str(ESLINT_FILE_PROCESSING_TIME_DEFAULT), - } + test_type = TsTestType.ESLINT + + from lib.nots.package_manager import constants + + peers = _create_pm(unit).get_peers_from_package_json() + deps = df.CustomDependencies.value5(unit, (peers,), {})[df.CustomDependencies.KEY].split() - _add_test(unit, "eslint", lint_files, deps, test_record, mod_dir) + if deps: + joined_deps = "\n".join(deps) + logger.info(f"{test_type} deps: \n{joined_deps}") + unit.ondepends(deps) + + flat_args = (test_type, "MODDIR") + + dart_record = create_dart_record( + TS_TEST_FIELDS_BASE + TS_TEST_SPECIFIC_FIELDS[test_type], + unit, + flat_args, + {}, + ) + dart_record[df.TestFiles.KEY] = test_files + dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME + + extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() + dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) + dart_record[df.LintFileProcessingTime.KEY] = str(ESLINT_FILE_PROCESSING_TIME_DEFAULT) + + data = ytest.dump_test(unit, dart_record) + if data: + unit.set_property(["DART_DATA", data]) unit.set(["TEST_RECIPES_VALUE", user_recipes]) -def _setup_tsc_typecheck(unit, tsconfig_paths: list[str]): +def _setup_tsc_typecheck(unit): if not _is_tests_enabled(unit): return if unit.get("_TS_TYPECHECK_VALUE") == "none": return - typecheck_files = get_values_list(unit, "TS_INPUT_FILES") - if not typecheck_files: + test_files = df.TestFiles.value7(unit, (), {})[df.TestFiles.KEY] + if not test_files: return + tsconfig_paths = unit.get("TS_CONFIG_PATH").split() tsconfig_path = tsconfig_paths[0] if len(tsconfig_paths) > 1: @@ -495,14 +503,36 @@ def _setup_tsc_typecheck(unit, tsconfig_paths: list[str]): unit.on_setup_install_node_modules_recipe() unit.on_setup_extract_output_tars_recipe([unit.get("MODDIR")]) - _add_test( + test_type = TsTestType.TSC_TYPECHECK + + from lib.nots.package_manager import constants + + peers = _create_pm(unit).get_peers_from_package_json() + deps = df.CustomDependencies.value5(unit, (peers,), {})[df.CustomDependencies.KEY].split() + + if deps: + joined_deps = "\n".join(deps) + logger.info(f"{test_type} deps: \n{joined_deps}") + unit.ondepends(deps) + + flat_args = (test_type,) + + dart_record = create_dart_record( + TS_TEST_FIELDS_BASE + TS_TEST_SPECIFIC_FIELDS[test_type], unit, - test_type="tsc_typecheck", - test_files=[resolve_common_const(f) for f in typecheck_files], - deps=_create_pm(unit).get_peers_from_package_json(), - test_record={"TS_CONFIG_PATH": tsconfig_path}, - test_cwd=unit.get("MODDIR"), + flat_args, + {}, ) + dart_record[df.TestFiles.KEY] = test_files + dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME + + extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() + dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) + dart_record[df.TsConfigPath.KEY] = tsconfig_path + + data = ytest.dump_test(unit, dart_record) + if data: + unit.set_property(["DART_DATA", data]) unit.set(["TEST_RECIPES_VALUE", user_recipes]) @@ -519,56 +549,6 @@ def _resolve_module_files(unit, mod_dir, file_paths): return resolved_files -def _add_test(unit, test_type, test_files, deps=None, test_record=None, test_cwd=None): - from lib.nots.package_manager import constants - - def sort_uniq(text): - return sorted(set(text)) - - recipes_lines = format_recipes(unit.get("TEST_RECIPES_VALUE")).strip().splitlines() - if recipes_lines: - deps = deps or [] - deps.extend([os.path.dirname(r.strip().split(" ")[0]) for r in recipes_lines]) - - if deps: - joined_deps = "\n".join(deps) - logger.info(f"{test_type} deps: \n{joined_deps}") - unit.ondepends(deps) - - test_dir = get_norm_unit_path(unit) - full_test_record = { - # Key to discover suite (see devtools/ya/test/explore/__init__.py#gen_suite) - "SCRIPT-REL-PATH": test_type, - # Test name as shown in PR check, should be unique inside one module - "TEST-NAME": test_type.lower().replace(".new", ""), - "TEST-TIMEOUT": unit.get("TEST_TIMEOUT") or "", - "TEST-ENV": ytest.prepare_env(unit.get("TEST_ENV_VALUE")), - "TESTED-PROJECT-NAME": os.path.splitext(unit.filename())[0], - "TEST-RECIPES": prepare_recipes(unit.get("TEST_RECIPES_VALUE")), - "SOURCE-FOLDER-PATH": test_dir, - "BUILD-FOLDER-PATH": test_dir, - "BINARY-PATH": os.path.join(test_dir, unit.filename()), - "SPLIT-FACTOR": unit.get("TEST_SPLIT_FACTOR") or "", - "FORK-MODE": unit.get("TEST_FORK_MODE") or "", - "SIZE": unit.get("TEST_SIZE_NAME") or "", - "TEST-DATA": serialize_list(get_values_list(unit, "TEST_DATA_VALUE")), - "TEST-FILES": serialize_list(test_files), - "TEST-CWD": test_cwd or "", - "TAG": serialize_list(get_values_list(unit, "TEST_TAGS_VALUE")), - "REQUIREMENTS": serialize_list(get_values_list(unit, "TEST_REQUIREMENTS_VALUE")), - "NODEJS-ROOT-VAR-NAME": unit.get("NODEJS-ROOT-VAR-NAME"), - "NODE-MODULES-BUNDLE-FILENAME": constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, - "CUSTOM-DEPENDENCIES": " ".join(sort_uniq((deps or []) + get_values_list(unit, "TEST_DEPENDS_VALUE"))), - } - - if test_record: - full_test_record.update(test_record) - - data = ytest.dump_test(unit, full_test_record) - if data: - unit.set_property(["DART_DATA", data]) - - def _set_resource_vars(unit, erm_json, tool, version, nodejs_major=None): # type: (any, ErmJsonLite, Version, str|None, int|None) -> None @@ -723,37 +703,54 @@ def on_ts_test_for_configure(unit, test_runner, default_config, node_modules_fil if unit.enabled('TS_COVERAGE'): unit.on_peerdir_ts_resource("nyc") - for_mod_path = unit.get("TS_TEST_FOR_PATH") + for_mod_path = df.TsTestForPath.value(unit, (), {})[df.TsTestForPath.KEY] unit.onpeerdir([for_mod_path]) unit.on_setup_extract_node_modules_recipe([for_mod_path]) unit.on_setup_extract_output_tars_recipe([for_mod_path]) - root = "$B" if test_runner == "hermione" else "$(BUILD_ROOT)" - unit.set(["TS_TEST_NM", os.path.join(root, for_mod_path, node_modules_filename)]) + build_root = "$B" if test_runner == TsTestType.HERMIONE else "$(BUILD_ROOT)" + unit.set(["TS_TEST_NM", os.path.join(build_root, for_mod_path, node_modules_filename)]) config_path = unit.get("TS_TEST_CONFIG_PATH") if not config_path: config_path = os.path.join(for_mod_path, default_config) unit.set(["TS_TEST_CONFIG_PATH", config_path]) - test_record = _add_ts_resources_to_test_record( - unit, - { - "TS-TEST-FOR-PATH": for_mod_path, - "TS-TEST-DATA-DIRS": serialize_list(_get_ts_test_data_dirs(unit)), - "TS-TEST-DATA-DIRS-RENAME": unit.get("_TS_TEST_DATA_DIRS_RENAME_VALUE"), - }, - ) - - test_files = get_values_list(unit, "_TS_TEST_SRCS_VALUE") - test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files) + test_files = df.TestFiles.value6(unit, (), {})[df.TestFiles.KEY] if not test_files: ymake.report_configure_error("No tests found") return - deps = _create_pm(unit).get_peers_from_package_json() - add_ts_test = _get_test_runner_handlers()[test_runner] - add_ts_test(unit, test_runner, test_files, deps, test_record) + from lib.nots.package_manager import constants + + peers = _create_pm(unit).get_peers_from_package_json() + deps = df.CustomDependencies.value5(unit, (peers,), {})[df.CustomDependencies.KEY].split() + + if deps: + joined_deps = "\n".join(deps) + logger.info(f"{test_runner} deps: \n{joined_deps}") + unit.ondepends(deps) + + flat_args = (test_runner, "TS_TEST_FOR_PATH") + spec_args = {'erm_json': _create_erm_json(unit)} + + dart_record = create_dart_record( + TS_TEST_FIELDS_BASE + TS_TEST_SPECIFIC_FIELDS[test_runner], + unit, + flat_args, + spec_args, + ) + dart_record[df.TestFiles.KEY] = test_files + dart_record[df.NodeModulesBundleFilename.KEY] = constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME + + extra_deps = df.CustomDependencies.value3(unit, (), {})[df.CustomDependencies.KEY].split() + dart_record[df.CustomDependencies.KEY] = " ".join(sort_uniq(deps + extra_deps)) + if test_runner == TsTestType.HERMIONE: + dart_record[df.Size.KEY] = "LARGE" + + data = ytest.dump_test(unit, dart_record) + if data: + unit.set_property(["DART_DATA", data]) @_with_report_configure_error @@ -776,16 +773,6 @@ def on_set_ts_test_for_vars(unit, for_mod): unit.set(["TS_TEST_FOR_PATH", rootrel_arc_src(for_mod, unit)]) -def _add_ts_resources_to_test_record(unit, test_record): - erm_json = _create_erm_json(unit) - for tool in erm_json.list_npm_packages(): - tool_resource_label = "{}-ROOT-VAR-NAME".format(tool.upper()) - tool_resource_value = unit.get(tool_resource_label) - if tool_resource_value: - test_record[tool_resource_label] = tool_resource_value - return test_record - - @_with_report_configure_error def on_ts_files(unit, *files): new_cmds = ['$COPY_CMD ${{input;context=TEXT:"{0}"}} ${{output;noauto:"{0}"}}'.format(f) for f in files] |