diff options
author | Aleksandr <ivansduck@gmail.com> | 2022-02-10 16:47:52 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:52 +0300 |
commit | ea6c5b7f172becca389cacaff7d5f45f6adccbe6 (patch) | |
tree | d16cef493ac1e092b4a03ab9437ec06ffe3d188f /build/plugins/ytest.py | |
parent | 37de222addabbef336dcaaea5f7c7645a629fc6d (diff) | |
download | ydb-ea6c5b7f172becca389cacaff7d5f45f6adccbe6.tar.gz |
Restoring authorship annotation for Aleksandr <ivansduck@gmail.com>. Commit 1 of 2.
Diffstat (limited to 'build/plugins/ytest.py')
-rw-r--r-- | build/plugins/ytest.py | 648 |
1 files changed, 324 insertions, 324 deletions
diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 8970837f0f..f58d00c99c 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -16,20 +16,20 @@ import collections import ymake -MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/' -MDS_SHEME = 'mds' -CANON_DATA_DIR_NAME = 'canondata' -CANON_OUTPUT_STORAGE = 'canondata_storage' -CANON_RESULT_FILE_NAME = 'result.json' -CANON_MDS_RESOURCE_REGEX = re.compile(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)') -CANON_SB_VAULT_REGEX = re.compile(r"\w+=(value|file):[-\w]+:\w+") -CANON_SBR_RESOURCE_REGEX = re.compile(r'(sbr:/?/?(\d+))') - +MDS_URI_PREFIX = 'https://storage.yandex-team.ru/get-devtools/' +MDS_SHEME = 'mds' +CANON_DATA_DIR_NAME = 'canondata' +CANON_OUTPUT_STORAGE = 'canondata_storage' +CANON_RESULT_FILE_NAME = 'result.json' +CANON_MDS_RESOURCE_REGEX = re.compile(re.escape(MDS_URI_PREFIX) + r'(.*?)($|#)') +CANON_SB_VAULT_REGEX = re.compile(r"\w+=(value|file):[-\w]+:\w+") +CANON_SBR_RESOURCE_REGEX = re.compile(r'(sbr:/?/?(\d+))') + VALID_NETWORK_REQUIREMENTS = ("full", "restricted") VALID_DNS_REQUIREMENTS = ("default", "local", "dns64") BLOCK_SEPARATOR = '=============================================================' -SPLIT_FACTOR_MAX_VALUE = 1000 -SPLIT_FACTOR_TEST_FILES_MAX_VALUE = 4250 +SPLIT_FACTOR_MAX_VALUE = 1000 +SPLIT_FACTOR_TEST_FILES_MAX_VALUE = 4250 PARTITION_MODS = ('SEQUENTIAL', 'MODULO') DEFAULT_TIDY_CONFIG = "build/config/tests/clang_tidy/config.yaml" DEFAULT_TIDY_CONFIG_MAP_PATH = "build/yandex_specific/config/clang_tidy/tidy_default_map.json" @@ -69,65 +69,65 @@ def is_yt_spec_contain_pool_info(filename): # XXX switch to yson in ymake + per return pool_re.search(yt_spec) and cypress_root_re.search(yt_spec) -def validate_sb_vault(name, value): - if not CANON_SB_VAULT_REGEX.match(value): - return "sb_vault value '{}' should follow pattern <ENV_NAME>=:<value|file>:<owner>:<vault key>".format(value) - - -def validate_numerical_requirement(name, value): - if mr.resolve_value(value) is None: - return "Cannot convert [[imp]]{}[[rst]] to the proper [[imp]]{}[[rst]] requirement value".format(value, name) - - -def validate_choice_requirement(name, val, valid): - if val not in valid: - return "Unknown [[imp]]{}[[rst]] requirement: [[imp]]{}[[rst]], choose from [[imp]]{}[[rst]]".format(name, val, ", ".join(valid)) - - +def validate_sb_vault(name, value): + if not CANON_SB_VAULT_REGEX.match(value): + return "sb_vault value '{}' should follow pattern <ENV_NAME>=:<value|file>:<owner>:<vault key>".format(value) + + +def validate_numerical_requirement(name, value): + if mr.resolve_value(value) is None: + return "Cannot convert [[imp]]{}[[rst]] to the proper [[imp]]{}[[rst]] requirement value".format(value, name) + + +def validate_choice_requirement(name, val, valid): + if val not in valid: + return "Unknown [[imp]]{}[[rst]] requirement: [[imp]]{}[[rst]], choose from [[imp]]{}[[rst]]".format(name, val, ", ".join(valid)) + + def validate_force_sandbox_requirement(name, value, test_size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run, check_func): if is_force_sandbox or not in_autocheck or is_fuzzing or is_ytexec_run: - if value == 'all': - return - return validate_numerical_requirement(name, value) - error_msg = validate_numerical_requirement(name, value) - if error_msg: - return error_msg + if value == 'all': + return + return validate_numerical_requirement(name, value) + error_msg = validate_numerical_requirement(name, value) + if error_msg: + return error_msg return check_func(mr.resolve_value(value), test_size, is_kvm) - - + + # TODO: Remove is_kvm param when there will be guarantees on RAM def validate_requirement(req_name, value, test_size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run): - req_checks = { - 'container': validate_numerical_requirement, + req_checks = { + 'container': validate_numerical_requirement, 'cpu': lambda n, v: validate_force_sandbox_requirement(n, v, test_size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run, reqs.check_cpu), - 'disk_usage': validate_numerical_requirement, - 'dns': lambda n, v: validate_choice_requirement(n, v, VALID_DNS_REQUIREMENTS), - 'kvm': None, - 'network': lambda n, v: validate_choice_requirement(n, v, VALID_NETWORK_REQUIREMENTS), + 'disk_usage': validate_numerical_requirement, + 'dns': lambda n, v: validate_choice_requirement(n, v, VALID_DNS_REQUIREMENTS), + 'kvm': None, + 'network': lambda n, v: validate_choice_requirement(n, v, VALID_NETWORK_REQUIREMENTS), 'ram': lambda n, v: validate_force_sandbox_requirement(n, v, test_size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run, reqs.check_ram), 'ram_disk': lambda n, v: validate_force_sandbox_requirement(n, v, test_size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run, reqs.check_ram_disk), - 'sb': None, - 'sb_vault': validate_sb_vault, - } - - if req_name not in req_checks: - return "Unknown requirement: [[imp]]{}[[rst]], choose from [[imp]]{}[[rst]]".format(req_name, ", ".join(sorted(req_checks))) - - if req_name in ('container', 'disk') and not is_force_sandbox: - return "Only [[imp]]LARGE[[rst]] tests without [[imp]]ya:force_distbuild[[rst]] tag can have [[imp]]{}[[rst]] requirement".format(req_name) - - check_func = req_checks[req_name] - if check_func: - return check_func(req_name, value) - - -def validate_test(unit, kw): + 'sb': None, + 'sb_vault': validate_sb_vault, + } + + if req_name not in req_checks: + return "Unknown requirement: [[imp]]{}[[rst]], choose from [[imp]]{}[[rst]]".format(req_name, ", ".join(sorted(req_checks))) + + if req_name in ('container', 'disk') and not is_force_sandbox: + return "Only [[imp]]LARGE[[rst]] tests without [[imp]]ya:force_distbuild[[rst]] tag can have [[imp]]{}[[rst]] requirement".format(req_name) + + check_func = req_checks[req_name] + if check_func: + return check_func(req_name, value) + + +def validate_test(unit, kw): def get_list(key): return deserialize_list(kw.get(key, "")) valid_kw = copy.deepcopy(kw) errors = [] - warnings = [] + warnings = [] if valid_kw.get('SCRIPT-REL-PATH') == 'boost.test': project_path = valid_kw.get('BUILD-FOLDER-PATH', "") @@ -143,66 +143,66 @@ def validate_test(unit, kw): size = valid_kw.get('SIZE', consts.TestSize.Small).lower() # TODO: use set instead list tags = get_list("TAG") - requirements_orig = get_list("REQUIREMENTS") + requirements_orig = get_list("REQUIREMENTS") in_autocheck = "ya:not_autocheck" not in tags and 'ya:manual' not in tags - is_fat = 'ya:fat' in tags - is_force_sandbox = 'ya:force_distbuild' not in tags and is_fat + is_fat = 'ya:fat' in tags + is_force_sandbox = 'ya:force_distbuild' not in tags and is_fat is_ytexec_run = 'ya:yt' in tags - is_fuzzing = valid_kw.get("FUZZING", False) - is_kvm = 'kvm' in requirements_orig + is_fuzzing = valid_kw.get("FUZZING", False) + is_kvm = 'kvm' in requirements_orig requirements = {} - list_requirements = ('sb_vault') - for req in requirements_orig: + list_requirements = ('sb_vault') + for req in requirements_orig: if req in ('kvm', ): requirements[req] = str(True) continue if ":" in req: req_name, req_value = req.split(":", 1) - if req_name in list_requirements: - requirements[req_name] = ",".join(filter(None, [requirements.get(req_name), req_value])) - else: - if req_name in requirements: - if req_value in ["0"]: - warnings.append("Requirement [[imp]]{}[[rst]] is dropped [[imp]]{}[[rst]] -> [[imp]]{}[[rst]]".format(req_name, requirements[req_name], req_value)) - del requirements[req_name] - elif requirements[req_name] != req_value: - warnings.append("Requirement [[imp]]{}[[rst]] is redefined [[imp]]{}[[rst]] -> [[imp]]{}[[rst]]".format(req_name, requirements[req_name], req_value)) - requirements[req_name] = req_value + if req_name in list_requirements: + requirements[req_name] = ",".join(filter(None, [requirements.get(req_name), req_value])) + else: + if req_name in requirements: + if req_value in ["0"]: + warnings.append("Requirement [[imp]]{}[[rst]] is dropped [[imp]]{}[[rst]] -> [[imp]]{}[[rst]]".format(req_name, requirements[req_name], req_value)) + del requirements[req_name] + elif requirements[req_name] != req_value: + warnings.append("Requirement [[imp]]{}[[rst]] is redefined [[imp]]{}[[rst]] -> [[imp]]{}[[rst]]".format(req_name, requirements[req_name], req_value)) + requirements[req_name] = req_value else: - requirements[req_name] = req_value + requirements[req_name] = req_value else: errors.append("Invalid requirement syntax [[imp]]{}[[rst]]: expect <requirement>:<value>".format(req)) - if not errors: - for req_name, req_value in requirements.items(): + if not errors: + for req_name, req_value in requirements.items(): error_msg = validate_requirement(req_name, req_value, size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run) - if error_msg: - errors += [error_msg] - + if error_msg: + errors += [error_msg] + invalid_requirements_for_distbuild = [requirement for requirement in requirements.keys() if requirement not in ('ram', 'ram_disk', 'cpu', 'network')] - sb_tags = [tag for tag in tags if tag.startswith('sb:')] + sb_tags = [tag for tag in tags if tag.startswith('sb:')] if is_fat: - if size != consts.TestSize.Large: - errors.append("Only LARGE test may have ya:fat tag") - + if size != consts.TestSize.Large: + errors.append("Only LARGE test may have ya:fat tag") + if in_autocheck and not is_force_sandbox: - if invalid_requirements_for_distbuild: - errors.append("'{}' REQUIREMENTS options can be used only for FAT tests without ya:force_distbuild tag. Remove TAG(ya:force_distbuild) or an option.".format(invalid_requirements_for_distbuild)) - if sb_tags: - errors.append("You can set sandbox tags '{}' only for FAT tests without ya:force_distbuild. Remove TAG(ya:force_sandbox) or sandbox tags.".format(sb_tags)) - if 'ya:sandbox_coverage' in tags: - errors.append("You can set 'ya:sandbox_coverage' tag only for FAT tests without ya:force_distbuild.") - else: + if invalid_requirements_for_distbuild: + errors.append("'{}' REQUIREMENTS options can be used only for FAT tests without ya:force_distbuild tag. Remove TAG(ya:force_distbuild) or an option.".format(invalid_requirements_for_distbuild)) + if sb_tags: + errors.append("You can set sandbox tags '{}' only for FAT tests without ya:force_distbuild. Remove TAG(ya:force_sandbox) or sandbox tags.".format(sb_tags)) + if 'ya:sandbox_coverage' in tags: + errors.append("You can set 'ya:sandbox_coverage' tag only for FAT tests without ya:force_distbuild.") + else: if is_force_sandbox: - errors.append('ya:force_sandbox can be used with LARGE tests only') - if 'ya:nofuse' in tags: - errors.append('ya:nofuse can be used with LARGE tests only') - if 'ya:privileged' in tags: - errors.append("ya:privileged can be used with LARGE tests only") - if in_autocheck and size == consts.TestSize.Large: - errors.append("LARGE test must have ya:fat tag") + errors.append('ya:force_sandbox can be used with LARGE tests only') + if 'ya:nofuse' in tags: + errors.append('ya:nofuse can be used with LARGE tests only') + if 'ya:privileged' in tags: + errors.append("ya:privileged can be used with LARGE tests only") + if in_autocheck and size == consts.TestSize.Large: + errors.append("LARGE test must have ya:fat tag") if 'ya:privileged' in tags and 'container' not in requirements: errors.append("Only tests with 'container' requirement can have 'ya:privileged' tag") @@ -236,34 +236,34 @@ def validate_test(unit, kw): valid_kw['REQUIREMENTS'] = serialize_list(requiremtens_list) if valid_kw.get("FUZZ-OPTS"): - for option in get_list("FUZZ-OPTS"): - if not option.startswith("-"): - errors.append("Unrecognized fuzzer option '[[imp]]{}[[rst]]'. All fuzzer options should start with '-'".format(option)) - break - eqpos = option.find("=") - if eqpos == -1 or len(option) == eqpos + 1: - errors.append("Unrecognized fuzzer option '[[imp]]{}[[rst]]'. All fuzzer options should obtain value specified after '='".format(option)) - break - if option[eqpos - 1] == " " or option[eqpos + 1] == " ": - errors.append("Spaces are not allowed: '[[imp]]{}[[rst]]'".format(option)) - break - if option[:eqpos] in ("-runs", "-dict", "-jobs", "-workers", "-artifact_prefix", "-print_final_stats"): - errors.append("You can't use '[[imp]]{}[[rst]]' - it will be automatically calculated or configured during run".format(option)) - break - - if valid_kw.get("YT-SPEC"): + for option in get_list("FUZZ-OPTS"): + if not option.startswith("-"): + errors.append("Unrecognized fuzzer option '[[imp]]{}[[rst]]'. All fuzzer options should start with '-'".format(option)) + break + eqpos = option.find("=") + if eqpos == -1 or len(option) == eqpos + 1: + errors.append("Unrecognized fuzzer option '[[imp]]{}[[rst]]'. All fuzzer options should obtain value specified after '='".format(option)) + break + if option[eqpos - 1] == " " or option[eqpos + 1] == " ": + errors.append("Spaces are not allowed: '[[imp]]{}[[rst]]'".format(option)) + break + if option[:eqpos] in ("-runs", "-dict", "-jobs", "-workers", "-artifact_prefix", "-print_final_stats"): + errors.append("You can't use '[[imp]]{}[[rst]]' - it will be automatically calculated or configured during run".format(option)) + break + + if valid_kw.get("YT-SPEC"): if not is_ytexec_run: - errors.append("You can use YT_SPEC macro only tests marked with ya:yt tag") - else: - for filename in get_list("YT-SPEC"): - filename = unit.resolve('$S/' + filename) - if not os.path.exists(filename): - errors.append("File '{}' specified in the YT_SPEC macro doesn't exist".format(filename)) - continue + errors.append("You can use YT_SPEC macro only tests marked with ya:yt tag") + else: + for filename in get_list("YT-SPEC"): + filename = unit.resolve('$S/' + filename) + if not os.path.exists(filename): + errors.append("File '{}' specified in the YT_SPEC macro doesn't exist".format(filename)) + continue if is_yt_spec_contain_pool_info(filename) and "ya:external" not in tags: tags.append("ya:external") tags.append("ya:yt_research_pool") - + if valid_kw.get("USE_ARCADIA_PYTHON") == "yes" and valid_kw.get("SCRIPT-REL-PATH") == "py.test": errors.append("PYTEST_SCRIPT is deprecated") @@ -274,23 +274,23 @@ def validate_test(unit, kw): if valid_kw.get('SPLIT-FACTOR'): if valid_kw.get('FORK-MODE') == 'none': errors.append('SPLIT_FACTOR must be use with FORK_TESTS() or FORK_SUBTESTS() macro') - - value = 1 + + value = 1 try: value = int(valid_kw.get('SPLIT-FACTOR')) if value <= 0: raise ValueError("must be > 0") - if value > SPLIT_FACTOR_MAX_VALUE: - raise ValueError("the maximum allowed value is {}".format(SPLIT_FACTOR_MAX_VALUE)) + if value > SPLIT_FACTOR_MAX_VALUE: + raise ValueError("the maximum allowed value is {}".format(SPLIT_FACTOR_MAX_VALUE)) except ValueError as e: errors.append('Incorrect SPLIT_FACTOR value: {}'.format(e)) - if valid_kw.get('FORK-TEST-FILES') and size != consts.TestSize.Large: - nfiles = count_entries(valid_kw.get('TEST-FILES')) - if nfiles * value > SPLIT_FACTOR_TEST_FILES_MAX_VALUE: - errors.append('Too much chunks generated:{} (limit: {}). Remove FORK_TEST_FILES() macro or reduce SPLIT_FACTOR({}).'.format( - nfiles * value, SPLIT_FACTOR_TEST_FILES_MAX_VALUE, value)) - + if valid_kw.get('FORK-TEST-FILES') and size != consts.TestSize.Large: + nfiles = count_entries(valid_kw.get('TEST-FILES')) + if nfiles * value > SPLIT_FACTOR_TEST_FILES_MAX_VALUE: + errors.append('Too much chunks generated:{} (limit: {}). Remove FORK_TEST_FILES() macro or reduce SPLIT_FACTOR({}).'.format( + nfiles * value, SPLIT_FACTOR_TEST_FILES_MAX_VALUE, value)) + unit_path = get_norm_unit_path(unit) if not is_fat and "ya:noretries" in tags and not is_ytexec_run \ and not unit_path.startswith("devtools/") \ @@ -300,25 +300,25 @@ def validate_test(unit, kw): and not unit_path.startswith("yp/tests"): errors.append("Only LARGE tests can have 'ya:noretries' tag") - if errors: - return None, warnings, errors - - return valid_kw, warnings, errors - - -def get_norm_unit_path(unit, extra=None): - path = _common.strip_roots(unit.path()) - if extra: - return '{}/{}'.format(path, extra) - return path - - -def dump_test(unit, kw): - valid_kw, warnings, errors = validate_test(unit, kw) - for w in warnings: - unit.message(['warn', w]) - for e in errors: - ymake.report_configure_error(e) + if errors: + return None, warnings, errors + + return valid_kw, warnings, errors + + +def get_norm_unit_path(unit, extra=None): + path = _common.strip_roots(unit.path()) + if extra: + return '{}/{}'.format(path, extra) + return path + + +def dump_test(unit, kw): + valid_kw, warnings, errors = validate_test(unit, kw) + for w in warnings: + unit.message(['warn', w]) + for e in errors: + ymake.report_configure_error(e) if valid_kw is None: return None string_handler = StringIO.StringIO() @@ -339,14 +339,14 @@ def deserialize_list(val): return filter(None, val.replace('"', "").split(";")) -def count_entries(x): - # see (de)serialize_list - assert x is None or isinstance(x, str), type(x) - if not x: - return 0 - return x.count(";") + 1 - - +def count_entries(x): + # see (de)serialize_list + assert x is None or isinstance(x, str), type(x) + if not x: + return 0 + return x.count(";") + 1 + + def get_values_list(unit, key): res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split()) return [r for r in res if r and r not in ['""', "''"]] @@ -357,31 +357,31 @@ def get_norm_paths(unit, key): return [x.rstrip('\\/') for x in get_values_list(unit, key)] -def get_unit_list_variable(unit, name): - items = unit.get(name) - if items: - items = items.split(' ') - assert items[0] == "${}".format(name), (items, name) - return items[1:] - return [] - - -def implies(a, b): - return bool((not a) or b) - - -def match_coverage_extractor_requirements(unit): - # we shouldn't add test if - return all([ - # tests are not requested - unit.get("TESTS_REQUESTED") == "yes", - # build doesn't imply clang coverage, which supports segment extraction from the binaries - unit.get("CLANG_COVERAGE") == "yes", - # contrib wasn't requested - implies(get_norm_unit_path(unit).startswith("contrib/"), unit.get("ENABLE_CONTRIB_COVERAGE") == "yes"), - ]) - - +def get_unit_list_variable(unit, name): + items = unit.get(name) + if items: + items = items.split(' ') + assert items[0] == "${}".format(name), (items, name) + return items[1:] + return [] + + +def implies(a, b): + return bool((not a) or b) + + +def match_coverage_extractor_requirements(unit): + # we shouldn't add test if + return all([ + # tests are not requested + unit.get("TESTS_REQUESTED") == "yes", + # build doesn't imply clang coverage, which supports segment extraction from the binaries + unit.get("CLANG_COVERAGE") == "yes", + # contrib wasn't requested + implies(get_norm_unit_path(unit).startswith("contrib/"), unit.get("ENABLE_CONTRIB_COVERAGE") == "yes"), + ]) + + def get_tidy_config_map(unit): global tidy_config_map if tidy_config_map is None: @@ -420,16 +420,16 @@ def onadd_ytest(unit, *args): test_data = sorted(_common.filter_out_by_keyword(spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED')) - if flat_args[1] == "fuzz.test": - unit.ondata("arcadia/fuzzing/{}/corpus.json".format(get_norm_unit_path(unit))) + if flat_args[1] == "fuzz.test": + unit.ondata("arcadia/fuzzing/{}/corpus.json".format(get_norm_unit_path(unit))) elif flat_args[1] == "go.test": data, _ = get_canonical_test_resources(unit) test_data += data - elif flat_args[1] == "coverage.extractor" and not match_coverage_extractor_requirements(unit): - # XXX - # Current ymake implementation doesn't allow to call macro inside the 'when' body - # that's why we add ADD_YTEST(coverage.extractor) to every PROGRAM entry and check requirements later - return + elif flat_args[1] == "coverage.extractor" and not match_coverage_extractor_requirements(unit): + # XXX + # Current ymake implementation doesn't allow to call macro inside the 'when' body + # that's why we add ADD_YTEST(coverage.extractor) to every PROGRAM entry and check requirements later + return elif flat_args[1] == "clang_tidy" and unit.get("TIDY") != "yes": # Graph is not prepared return @@ -439,7 +439,7 @@ def onadd_ytest(unit, *args): test_tags = serialize_list(_get_test_tags(unit, spec_args)) test_timeout = ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or '' test_requirements = spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE') - + if flat_args[1] != "clang_tidy" and unit.get("TIDY") == "yes": # graph changed for clang_tidy tests if flat_args[1] in ("unittest.py", "gunittest", "g_benchmark"): @@ -471,17 +471,17 @@ def onadd_ytest(unit, *args): fork_mode = fork_mode or spec_args.get('FORK_MODE', []) or unit.get('TEST_FORK_MODE').split() fork_mode = ' '.join(fork_mode) if fork_mode else '' - unit_path = get_norm_unit_path(unit) - + unit_path = get_norm_unit_path(unit) + test_record = { 'TEST-NAME': flat_args[0], 'SCRIPT-REL-PATH': flat_args[1], 'TESTED-PROJECT-NAME': unit.name(), 'TESTED-PROJECT-FILENAME': unit.filename(), - 'SOURCE-FOLDER-PATH': unit_path, - # TODO get rid of BUILD-FOLDER-PATH - 'BUILD-FOLDER-PATH': unit_path, - 'BINARY-PATH': "{}/{}".format(unit_path, unit.filename()), + 'SOURCE-FOLDER-PATH': unit_path, + # TODO get rid of BUILD-FOLDER-PATH + 'BUILD-FOLDER-PATH': unit_path, + 'BINARY-PATH': "{}/{}".format(unit_path, unit.filename()), 'GLOBAL-LIBRARY-PATH': unit.global_filename(), 'CUSTOM-DEPENDENCIES': ' '.join(spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE')), 'TEST-RECIPES': prepare_recipes(unit.get("TEST_RECIPES_VALUE")), @@ -495,9 +495,9 @@ def onadd_ytest(unit, *args): 'TAG': test_tags, 'REQUIREMENTS': serialize_list(test_requirements), 'TEST-CWD': unit.get('TEST_CWD_VALUE') or '', - 'FUZZ-DICTS': serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')), - 'FUZZ-OPTS': serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')), - 'YT-SPEC': serialize_list(spec_args.get('YT_SPEC', []) + get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')), + 'FUZZ-DICTS': serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')), + 'FUZZ-OPTS': serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')), + 'YT-SPEC': serialize_list(spec_args.get('YT_SPEC', []) + get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')), 'BLOB': unit.get('TEST_BLOB_DATA') or '', 'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '', 'TEST_IOS_DEVICE_TYPE': unit.get('TEST_IOS_DEVICE_TYPE_VALUE') or '', @@ -513,12 +513,12 @@ def onadd_ytest(unit, *args): else: test_record["TEST-NAME"] += "_bench" - if flat_args[1] == 'fuzz.test' and unit.get('FUZZING') == 'yes': - test_record['FUZZING'] = '1' + if flat_args[1] == 'fuzz.test' and unit.get('FUZZING') == 'yes': + test_record['FUZZING'] = '1' # use all cores if fuzzing requested test_record['REQUIREMENTS'] = serialize_list(filter(None, deserialize_list(test_record['REQUIREMENTS']) + ["cpu:all", "ram:all"])) - - data = dump_test(unit, test_record) + + data = dump_test(unit, test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data) @@ -550,7 +550,7 @@ def onadd_check(unit, *args): flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1}, args) check_type = flat_args[0] - test_dir = get_norm_unit_path(unit) + test_dir = get_norm_unit_path(unit) test_timeout = '' fork_mode = '' @@ -558,7 +558,7 @@ def onadd_check(unit, *args): extra_test_dart_data = {} ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes' - if check_type in ["flake8.py2", "flake8.py3"]: + if check_type in ["flake8.py2", "flake8.py3"]: script_rel_path = check_type fork_mode = unit.get('TEST_FORK_MODE') or '' elif check_type == "JAVA_STYLE": @@ -628,18 +628,18 @@ def onadd_check(unit, *args): 'FORK-TEST-FILES': '', 'SIZE': 'SMALL', 'TAG': '', - 'REQUIREMENTS': '', + 'REQUIREMENTS': '', 'USE_ARCADIA_PYTHON': use_arcadia_python or '', 'OLD_PYTEST': 'no', 'PYTHON-PATHS': '', - # TODO remove FILES, see DEVTOOLS-7052 - 'FILES': test_files, - 'TEST-FILES': test_files, + # TODO remove FILES, see DEVTOOLS-7052 + 'FILES': test_files, + 'TEST-FILES': test_files, 'NO_JBUILD': 'yes' if ymake_java_test else 'no', } test_record.update(extra_test_dart_data) - data = dump_test(unit, test_record) + data = dump_test(unit, test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data) @@ -659,10 +659,10 @@ def onadd_check_py_imports(unit, *args): return unit.onpeerdir(['library/python/testing/import_test']) check_type = "py.imports" - test_dir = get_norm_unit_path(unit) + test_dir = get_norm_unit_path(unit) use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') - test_files = serialize_list([get_norm_unit_path(unit, unit.filename())]) + test_files = serialize_list([get_norm_unit_path(unit, unit.filename())]) test_record = { 'TEST-NAME': "pyimports", 'TEST-TIMEOUT': '', @@ -681,15 +681,15 @@ def onadd_check_py_imports(unit, *args): 'USE_ARCADIA_PYTHON': use_arcadia_python or '', 'OLD_PYTEST': 'no', 'PYTHON-PATHS': '', - # TODO remove FILES, see DEVTOOLS-7052 - 'FILES': test_files, - 'TEST-FILES': test_files, + # TODO remove FILES, see DEVTOOLS-7052 + 'FILES': test_files, + 'TEST-FILES': test_files, } if unit.get('NO_CHECK_IMPORTS_FOR_VALUE') != "None": test_record["NO-CHECK"] = serialize_list(get_values_list(unit, 'NO_CHECK_IMPORTS_FOR_VALUE') or ["*"]) else: test_record["NO-CHECK"] = '' - data = dump_test(unit, test_record) + data = dump_test(unit, test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data) @@ -714,14 +714,14 @@ def onadd_pytest_script(unit, *args): test_files = get_values_list(unit, 'TEST_SRCS_VALUE') tags = _get_test_tags(unit) - requirements = get_values_list(unit, 'TEST_REQUIREMENTS_VALUE') + requirements = get_values_list(unit, 'TEST_REQUIREMENTS_VALUE') test_data = get_norm_paths(unit, 'TEST_DATA_VALUE') - data, data_files = get_canonical_test_resources(unit) + data, data_files = get_canonical_test_resources(unit) test_data += data python_paths = get_values_list(unit, 'TEST_PYTHON_PATH_VALUE') binary_path = None test_cwd = unit.get('TEST_CWD_VALUE') or '' - _dump_test(unit, test_type, test_files, timeout, get_norm_unit_path(unit), custom_deps, test_data, python_paths, split_factor, fork_mode, test_size, tags, requirements, binary_path, test_cwd=test_cwd, data_files=data_files) + _dump_test(unit, test_type, test_files, timeout, get_norm_unit_path(unit), custom_deps, test_data, python_paths, split_factor, fork_mode, test_size, tags, requirements, binary_path, test_cwd=test_cwd, data_files=data_files) def onadd_pytest_bin(unit, *args): @@ -756,40 +756,40 @@ def add_test_to_dart(unit, test_type, binary_path=None, runner_bin=None): test_size = unit.get('TEST_SIZE_NAME') or '' test_cwd = unit.get('TEST_CWD_VALUE') or '' - unit_path = unit.path() + unit_path = unit.path() test_files = get_values_list(unit, 'TEST_SRCS_VALUE') tags = _get_test_tags(unit) - requirements = get_values_list(unit, 'TEST_REQUIREMENTS_VALUE') + requirements = get_values_list(unit, 'TEST_REQUIREMENTS_VALUE') test_data = get_norm_paths(unit, 'TEST_DATA_VALUE') - data, data_files = get_canonical_test_resources(unit) + data, data_files = get_canonical_test_resources(unit) test_data += data python_paths = get_values_list(unit, 'TEST_PYTHON_PATH_VALUE') - yt_spec = get_values_list(unit, 'TEST_YT_SPEC_VALUE') + yt_spec = get_values_list(unit, 'TEST_YT_SPEC_VALUE') if not binary_path: - binary_path = os.path.join(unit_path, unit.filename()) - _dump_test(unit, test_type, test_files, timeout, get_norm_unit_path(unit), custom_deps, test_data, python_paths, split_factor, fork_mode, test_size, tags, requirements, binary_path, test_cwd=test_cwd, runner_bin=runner_bin, yt_spec=yt_spec, data_files=data_files) + binary_path = os.path.join(unit_path, unit.filename()) + _dump_test(unit, test_type, test_files, timeout, get_norm_unit_path(unit), custom_deps, test_data, python_paths, split_factor, fork_mode, test_size, tags, requirements, binary_path, test_cwd=test_cwd, runner_bin=runner_bin, yt_spec=yt_spec, data_files=data_files) def extract_java_system_properties(unit, args): if len(args) % 2: - return [], 'Wrong use of SYSTEM_PROPERTIES in {}: odd number of arguments'.format(unit.path()) + return [], 'Wrong use of SYSTEM_PROPERTIES in {}: odd number of arguments'.format(unit.path()) - props = [] + props = [] for x, y in zip(args[::2], args[1::2]): if x == 'FILE': if y.startswith('${BINDIR}') or y.startswith('${ARCADIA_BUILD_ROOT}') or y.startswith('/'): - return [], 'Wrong use of SYSTEM_PROPERTIES in {}: absolute/build file path {}'.format(unit.path(), y) + return [], 'Wrong use of SYSTEM_PROPERTIES in {}: absolute/build file path {}'.format(unit.path(), y) y = _common.rootrel_arc_src(y, unit) if not os.path.exists(unit.resolve('$S/' + y)): - return [], 'Wrong use of SYSTEM_PROPERTIES in {}: can\'t resolve {}'.format(unit.path(), y) + return [], 'Wrong use of SYSTEM_PROPERTIES in {}: can\'t resolve {}'.format(unit.path(), y) y = '${ARCADIA_ROOT}/' + y props.append({'type': 'file', 'path': y}) else: props.append({'type': 'inline', 'key': x, 'value': y}) - return props, None + return props, None def onjava_test(unit, *args): @@ -801,28 +801,28 @@ def onjava_test(unit, *args): if unit.get('MODULE_TYPE') == 'JTEST_FOR': if not unit.get('UNITTEST_DIR'): - ymake.report_configure_error('skip JTEST_FOR in {}: no args provided'.format(unit.path())) - return + ymake.report_configure_error('skip JTEST_FOR in {}: no args provided'.format(unit.path())) + return java_cp_arg_type = unit.get('JAVA_CLASSPATH_CMD_TYPE_VALUE') or 'MANIFEST' if java_cp_arg_type not in ('MANIFEST', 'COMMAND_FILE', 'LIST'): ymake.report_configure_error('{}: TEST_JAVA_CLASSPATH_CMD_TYPE({}) are invalid. Choose argument from MANIFEST, COMMAND_FILE or LIST)'.format(unit.path(), java_cp_arg_type)) return - unit_path = unit.path() - path = _common.strip_roots(unit_path) + unit_path = unit.path() + path = _common.strip_roots(unit_path) test_data = get_norm_paths(unit, 'TEST_DATA_VALUE') - test_data.append('arcadia/build/scripts/run_junit.py') + test_data.append('arcadia/build/scripts/run_junit.py') test_data.append('arcadia/build/scripts/unpacking_jtest_runner.py') - data, data_files = get_canonical_test_resources(unit) - test_data += data - - props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE')) - if error_mgs: - ymake.report_configure_error(error_mgs) - return + data, data_files = get_canonical_test_resources(unit) + test_data += data + + props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE')) + if error_mgs: + ymake.report_configure_error(error_mgs) + return for prop in props: if prop['type'] == 'file': test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia')) @@ -831,7 +831,7 @@ def onjava_test(unit, *args): test_cwd = unit.get('TEST_CWD_VALUE') or '' # TODO: validate test_cwd value - if unit.get('MODULE_TYPE') == 'JUNIT5': + if unit.get('MODULE_TYPE') == 'JUNIT5': script_rel_path = 'junit5.test' else: script_rel_path = 'junit.test' @@ -842,7 +842,7 @@ def onjava_test(unit, *args): 'TEST-NAME': '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)]), 'SCRIPT-REL-PATH': script_rel_path, 'TEST-TIMEOUT': unit.get('TEST_TIMEOUT') or '', - 'TESTED-PROJECT-NAME': path, + 'TESTED-PROJECT-NAME': path, 'TEST-ENV': prepare_env(unit.get("TEST_ENV_VALUE")), # 'TEST-PRESERVE-ENV': 'da', 'TEST-DATA': serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED'))), @@ -878,7 +878,7 @@ def onjava_test(unit, *args): else: test_record['TEST_JAR'] = '{}/{}.jar'.format(unit.get('MODDIR'), unit.get('REALPRJNAME')) - data = dump_test(unit, test_record) + data = dump_test(unit, test_record) if data: unit.set_property(['DART_DATA', data]) @@ -892,7 +892,7 @@ def onjava_test_deps(unit, *args): assert len(args) == 1 mode = args[0] - path = get_norm_unit_path(unit) + path = get_norm_unit_path(unit) ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes' test_record = { @@ -923,7 +923,7 @@ def onjava_test_deps(unit, *args): if ymake_java_test: test_record['CLASSPATH'] = '$B/{}/{}.jar ${{DART_CLASSPATH}}'.format(unit.get('MODDIR'), unit.get('REALPRJNAME')) - data = dump_test(unit, test_record) + data = dump_test(unit, test_record) unit.set_property(['DART_DATA', data]) @@ -951,12 +951,12 @@ def _dump_test( fork_mode, test_size, tags, - requirements, + requirements, binary_path='', old_pytest=False, test_cwd=None, - runner_bin=None, - yt_spec=None, + runner_bin=None, + yt_spec=None, data_files=None ): @@ -965,7 +965,7 @@ def _dump_test( else: script_rel_path = test_type - unit_path = unit.path() + unit_path = unit.path() fork_test_files = unit.get('FORK_TEST_FILES_MODE') fork_mode = ' '.join(fork_mode) if fork_mode else '' use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') @@ -1029,11 +1029,11 @@ def onrun(unit, *args): def onsetup_exectest(unit, *args): - command = unit.get(["EXECTEST_COMMAND_VALUE"]) - if command is None: - ymake.report_configure_error("EXECTEST must have at least one RUN macro") - return - command = command.replace("$EXECTEST_COMMAND_VALUE", "") + command = unit.get(["EXECTEST_COMMAND_VALUE"]) + if command is None: + ymake.report_configure_error("EXECTEST must have at least one RUN macro") + return + command = command.replace("$EXECTEST_COMMAND_VALUE", "") if "PYTHON_BIN" in command: unit.ondepends('contrib/tools/python') unit.set(["TEST_BLOB_DATA", base64.b64encode(command)]) @@ -1043,71 +1043,71 @@ def onsetup_exectest(unit, *args): def onsetup_run_python(unit): if unit.get("USE_ARCADIA_PYTHON") == "yes": unit.ondepends('contrib/tools/python') - - -def get_canonical_test_resources(unit): - unit_path = unit.path() + + +def get_canonical_test_resources(unit): + unit_path = unit.path() canon_data_dir = os.path.join(unit.resolve(unit_path), CANON_DATA_DIR_NAME, unit.get('CANONIZE_SUB_PATH') or '') - - try: - _, dirs, files = next(os.walk(canon_data_dir)) - except StopIteration: - # path doesn't exist - return [], [] - - if CANON_RESULT_FILE_NAME in files: - return _get_canonical_data_resources_v2(os.path.join(canon_data_dir, CANON_RESULT_FILE_NAME), unit_path) - return [], [] - - -def _load_canonical_file(filename, unit_path): - try: - with open(filename) as results_file: - return json.load(results_file) - except Exception as e: - print>>sys.stderr, "malformed canonical data in {}: {} ({})".format(unit_path, e, filename) - return {} - - -def _get_resource_from_uri(uri): - m = CANON_MDS_RESOURCE_REGEX.match(uri) - if m: - res_id = m.group(1) - return "{}:{}".format(MDS_SHEME, res_id) - - m = CANON_SBR_RESOURCE_REGEX.match(uri) - if m: - # There might be conflict between resources, because all resources in sandbox have 'resource.tar.gz' name - # That's why we use notation with '=' to specify specific path for resource - uri = m.group(1) - res_id = m.group(2) - return "{}={}".format(uri, '/'.join([CANON_OUTPUT_STORAGE, res_id])) - - -def _get_external_resources_from_canon_data(data): - # Method should work with both canonization versions: - # result.json: {'uri':X 'checksum':Y} - # result.json: {'testname': {'uri':X 'checksum':Y}} - # result.json: {'testname': [{'uri':X 'checksum':Y}]} - # Also there is a bug - if user returns {'uri': 1} from test - machinery will fail - # That's why we check 'uri' and 'checksum' fields presence - # (it's still a bug - user can return {'uri':X, 'checksum': Y}, we need to unify canonization format) - res = set() - - if isinstance(data, dict): - if 'uri' in data and 'checksum' in data: - resource = _get_resource_from_uri(data['uri']) - if resource: - res.add(resource) - else: - for k, v in data.iteritems(): - res.update(_get_external_resources_from_canon_data(v)) - elif isinstance(data, list): - for e in data: - res.update(_get_external_resources_from_canon_data(e)) - - return res - - -def _get_canonical_data_resources_v2(filename, unit_path): + + try: + _, dirs, files = next(os.walk(canon_data_dir)) + except StopIteration: + # path doesn't exist + return [], [] + + if CANON_RESULT_FILE_NAME in files: + return _get_canonical_data_resources_v2(os.path.join(canon_data_dir, CANON_RESULT_FILE_NAME), unit_path) + return [], [] + + +def _load_canonical_file(filename, unit_path): + try: + with open(filename) as results_file: + return json.load(results_file) + except Exception as e: + print>>sys.stderr, "malformed canonical data in {}: {} ({})".format(unit_path, e, filename) + return {} + + +def _get_resource_from_uri(uri): + m = CANON_MDS_RESOURCE_REGEX.match(uri) + if m: + res_id = m.group(1) + return "{}:{}".format(MDS_SHEME, res_id) + + m = CANON_SBR_RESOURCE_REGEX.match(uri) + if m: + # There might be conflict between resources, because all resources in sandbox have 'resource.tar.gz' name + # That's why we use notation with '=' to specify specific path for resource + uri = m.group(1) + res_id = m.group(2) + return "{}={}".format(uri, '/'.join([CANON_OUTPUT_STORAGE, res_id])) + + +def _get_external_resources_from_canon_data(data): + # Method should work with both canonization versions: + # result.json: {'uri':X 'checksum':Y} + # result.json: {'testname': {'uri':X 'checksum':Y}} + # result.json: {'testname': [{'uri':X 'checksum':Y}]} + # Also there is a bug - if user returns {'uri': 1} from test - machinery will fail + # That's why we check 'uri' and 'checksum' fields presence + # (it's still a bug - user can return {'uri':X, 'checksum': Y}, we need to unify canonization format) + res = set() + + if isinstance(data, dict): + if 'uri' in data and 'checksum' in data: + resource = _get_resource_from_uri(data['uri']) + if resource: + res.add(resource) + else: + for k, v in data.iteritems(): + res.update(_get_external_resources_from_canon_data(v)) + elif isinstance(data, list): + for e in data: + res.update(_get_external_resources_from_canon_data(e)) + + return res + + +def _get_canonical_data_resources_v2(filename, unit_path): return (_get_external_resources_from_canon_data(_load_canonical_file(filename, unit_path)), [filename]) |