diff options
author | akastornov <akastornov@yandex-team.ru> | 2022-02-10 16:46:03 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:03 +0300 |
commit | 8d3a5ed3a188a34167eaee54f1691ce5c9edf2f3 (patch) | |
tree | 1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /build | |
parent | 3a2de774d91ca8d7325aaf81c200b1d2047725e6 (diff) | |
download | ydb-8d3a5ed3a188a34167eaee54f1691ce5c9edf2f3.tar.gz |
Restoring authorship annotation for <akastornov@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'build')
32 files changed, 882 insertions, 882 deletions
diff --git a/build/plugins/java.py b/build/plugins/java.py index 309d98bc9d..16fc126734 100644 --- a/build/plugins/java.py +++ b/build/plugins/java.py @@ -1,43 +1,43 @@ -import _common as common +import _common as common import ymake -import json +import json import os -import base64 - - -DELIM = '================================' - - -def split_args(s): # TODO quotes, escapes - return filter(None, s.split()) - - -def extract_macro_calls(unit, macro_value_name, macro_calls_delim): - if not unit.get(macro_value_name): - return [] - - return filter(None, map(split_args, unit.get(macro_value_name).replace('$' + macro_value_name, '').split(macro_calls_delim))) - - -def extract_macro_calls2(unit, macro_value_name): - if not unit.get(macro_value_name): - return [] - - calls = [] - for call_encoded_args in unit.get(macro_value_name).strip().split(): - call_args = json.loads(base64.b64decode(call_encoded_args), encoding='utf-8') - calls.append(call_args) - - return calls - - +import base64 + + +DELIM = '================================' + + +def split_args(s): # TODO quotes, escapes + return filter(None, s.split()) + + +def extract_macro_calls(unit, macro_value_name, macro_calls_delim): + if not unit.get(macro_value_name): + return [] + + return filter(None, map(split_args, unit.get(macro_value_name).replace('$' + macro_value_name, '').split(macro_calls_delim))) + + +def extract_macro_calls2(unit, macro_value_name): + if not unit.get(macro_value_name): + return [] + + calls = [] + for call_encoded_args in unit.get(macro_value_name).strip().split(): + call_args = json.loads(base64.b64decode(call_encoded_args), encoding='utf-8') + calls.append(call_args) + + return calls + + def on_run_jbuild_program(unit, *args): args = list(args) """ Custom code generation @link: https://wiki.yandex-team.ru/yatool/java/#kodogeneracijarunjavaprogram """ - + flat, kv = common.sort_by_keywords({'IN': -1, 'IN_DIR': -1, 'OUT': -1, 'OUT_DIR': -1, 'CWD': 1, 'CLASSPATH': -1, 'CP_USE_COMMAND_FILE': 1, 'ADD_SRCS_TO_CLASSPATH': 0}, args) depends = kv.get('CLASSPATH', []) + kv.get('JAR', []) fake_out = None @@ -45,18 +45,18 @@ def on_run_jbuild_program(unit, *args): # XXX: hack to force ymake to build dependencies fake_out = "fake.out.{}".format(hash(tuple(args))) unit.on_run_java(['TOOL'] + depends + ["OUT", fake_out]) - + if not kv.get('CP_USE_COMMAND_FILE'): args += ['CP_USE_COMMAND_FILE', unit.get(['JAVA_PROGRAM_CP_USE_COMMAND_FILE']) or 'yes'] if fake_out is not None: args += ['FAKE_OUT', fake_out] - prev = unit.get(['RUN_JAVA_PROGRAM_VALUE']) or '' - new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() - unit.set(['RUN_JAVA_PROGRAM_VALUE', new_val]) - - + prev = unit.get(['RUN_JAVA_PROGRAM_VALUE']) or '' + new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() + unit.set(['RUN_JAVA_PROGRAM_VALUE', new_val]) + + def ongenerate_script(unit, *args): """ heretic@ promised to make tutorial here @@ -74,48 +74,48 @@ def ongenerate_script(unit, *args): unit.set(['GENERATE_SCRIPT_VALUE', new_val]) -def onjava_module(unit, *args): - args_delim = unit.get('ARGS_DELIM') +def onjava_module(unit, *args): + args_delim = unit.get('ARGS_DELIM') idea_only = True if 'IDEA_ONLY' in args else False - + if idea_only: if unit.get('YA_IDE_IDEA') != 'yes': return if unit.get('YMAKE_JAVA_MODULES') != 'yes': return - data = { + data = { 'BUNDLE_NAME': unit.name(), - 'PATH': unit.path(), + 'PATH': unit.path(), 'IDEA_ONLY': 'yes' if idea_only else 'no', - 'MODULE_TYPE': unit.get('MODULE_TYPE'), - 'MODULE_ARGS': unit.get('MODULE_ARGS'), + 'MODULE_TYPE': unit.get('MODULE_TYPE'), + 'MODULE_ARGS': unit.get('MODULE_ARGS'), 'MANAGED_PEERS': '${MANAGED_PEERS}', 'MANAGED_PEERS_CLOSURE': '${MANAGED_PEERS_CLOSURE}', 'NON_NAMAGEABLE_PEERS': '${NON_NAMAGEABLE_PEERS}', 'TEST_CLASSPATH_MANAGED': '${TEST_CLASSPATH_MANAGED}', - 'EXCLUDE': extract_macro_calls(unit, 'EXCLUDE_VALUE', args_delim), - 'JAVA_SRCS': extract_macro_calls(unit, 'JAVA_SRCS_VALUE', args_delim), - 'JAVAC_FLAGS': extract_macro_calls(unit, 'JAVAC_FLAGS_VALUE', args_delim), - 'ANNOTATION_PROCESSOR': extract_macro_calls(unit, 'ANNOTATION_PROCESSOR_VALUE', args_delim), - 'EXTERNAL_JAR': extract_macro_calls(unit, 'EXTERNAL_JAR_VALUE', args_delim), - 'RUN_JAVA_PROGRAM': extract_macro_calls2(unit, 'RUN_JAVA_PROGRAM_VALUE'), + 'EXCLUDE': extract_macro_calls(unit, 'EXCLUDE_VALUE', args_delim), + 'JAVA_SRCS': extract_macro_calls(unit, 'JAVA_SRCS_VALUE', args_delim), + 'JAVAC_FLAGS': extract_macro_calls(unit, 'JAVAC_FLAGS_VALUE', args_delim), + 'ANNOTATION_PROCESSOR': extract_macro_calls(unit, 'ANNOTATION_PROCESSOR_VALUE', args_delim), + 'EXTERNAL_JAR': extract_macro_calls(unit, 'EXTERNAL_JAR_VALUE', args_delim), + 'RUN_JAVA_PROGRAM': extract_macro_calls2(unit, 'RUN_JAVA_PROGRAM_VALUE'), 'RUN_JAVA_PROGRAM_MANAGED': '${RUN_JAVA_PROGRAM_MANAGED}', 'MAVEN_GROUP_ID': extract_macro_calls(unit, 'MAVEN_GROUP_ID_VALUE', args_delim), 'JAR_INCLUDE_FILTER': extract_macro_calls(unit, 'JAR_INCLUDE_FILTER_VALUE', args_delim), 'JAR_EXCLUDE_FILTER': extract_macro_calls(unit, 'JAR_EXCLUDE_FILTER_VALUE', args_delim), - - # TODO remove when java test dart is in prod - 'UNITTEST_DIR': unit.get('UNITTEST_DIR'), - 'SYSTEM_PROPERTIES': extract_macro_calls(unit, 'SYSTEM_PROPERTIES_VALUE', args_delim), - 'JVM_ARGS': extract_macro_calls(unit, 'JVM_ARGS_VALUE', args_delim), - 'TEST_CWD': extract_macro_calls(unit, 'TEST_CWD_VALUE', args_delim), - 'TEST_DATA': extract_macro_calls(unit, '__test_data', args_delim), - 'TEST_FORK_MODE': extract_macro_calls(unit, 'TEST_FORK_MODE', args_delim), - 'SPLIT_FACTOR': extract_macro_calls(unit, 'TEST_SPLIT_FACTOR', args_delim), - 'TIMEOUT': extract_macro_calls(unit, 'TEST_TIMEOUT', args_delim), - 'TAG': extract_macro_calls(unit, 'TEST_TAGS_VALUE', args_delim), - 'SIZE': extract_macro_calls(unit, 'TEST_SIZE_NAME', args_delim), + + # TODO remove when java test dart is in prod + 'UNITTEST_DIR': unit.get('UNITTEST_DIR'), + 'SYSTEM_PROPERTIES': extract_macro_calls(unit, 'SYSTEM_PROPERTIES_VALUE', args_delim), + 'JVM_ARGS': extract_macro_calls(unit, 'JVM_ARGS_VALUE', args_delim), + 'TEST_CWD': extract_macro_calls(unit, 'TEST_CWD_VALUE', args_delim), + 'TEST_DATA': extract_macro_calls(unit, '__test_data', args_delim), + 'TEST_FORK_MODE': extract_macro_calls(unit, 'TEST_FORK_MODE', args_delim), + 'SPLIT_FACTOR': extract_macro_calls(unit, 'TEST_SPLIT_FACTOR', args_delim), + 'TIMEOUT': extract_macro_calls(unit, 'TEST_TIMEOUT', args_delim), + 'TAG': extract_macro_calls(unit, 'TEST_TAGS_VALUE', args_delim), + 'SIZE': extract_macro_calls(unit, 'TEST_SIZE_NAME', args_delim), 'DEPENDS': extract_macro_calls(unit, 'TEST_DEPENDS_VALUE', args_delim), 'IDEA_EXCLUDE': extract_macro_calls(unit, 'IDEA_EXCLUDE_DIRS_VALUE', args_delim), 'IDEA_RESOURCE': extract_macro_calls(unit, 'IDEA_RESOURCE_DIRS_VALUE', args_delim), @@ -125,17 +125,17 @@ def onjava_module(unit, *args): 'TEST_DATA': extract_macro_calls(unit, 'TEST_DATA_VALUE', args_delim), 'JAVA_FORBIDDEN_LIBRARIES': extract_macro_calls(unit, 'JAVA_FORBIDDEN_LIBRARIES_VALUE', args_delim), 'JDK_RESOURCE': 'JDK' + (unit.get('JDK_VERSION') or '_DEFAULT') - } + } if unit.get('ENABLE_PREVIEW_VALUE') == 'yes' and unit.get('JDK_VERSION') in ('15', '16', '17'): data['ENABLE_PREVIEW'] = extract_macro_calls(unit, 'ENABLE_PREVIEW_VALUE', args_delim) if unit.get('SAVE_JAVAC_GENERATED_SRCS_DIR') and unit.get('SAVE_JAVAC_GENERATED_SRCS_TAR'): data['SAVE_JAVAC_GENERATED_SRCS_DIR'] = extract_macro_calls(unit, 'SAVE_JAVAC_GENERATED_SRCS_DIR', args_delim) data['SAVE_JAVAC_GENERATED_SRCS_TAR'] = extract_macro_calls(unit, 'SAVE_JAVAC_GENERATED_SRCS_TAR', args_delim) - + if unit.get('JAVA_ADD_DLLS_VALUE') == 'yes': data['ADD_DLLS_FROM_DEPENDS'] = extract_macro_calls(unit, 'JAVA_ADD_DLLS_VALUE', args_delim) - + if unit.get('ERROR_PRONE_VALUE') == 'yes': data['ERROR_PRONE'] = extract_macro_calls(unit, 'ERROR_PRONE_VALUE', args_delim) @@ -189,36 +189,36 @@ def onjava_module(unit, *args): if macro_str and macro_str == 'yes': data['VCS_INFO_DISABLE_CACHE__NO_UID__'] = macro_val - for java_srcs_args in data['JAVA_SRCS']: - external = None - - for i in xrange(len(java_srcs_args)): - arg = java_srcs_args[i] - - if arg == 'EXTERNAL': - if not i + 1 < len(java_srcs_args): - continue # TODO configure error - - ex = java_srcs_args[i + 1] - - if ex in ('EXTERNAL', 'SRCDIR', 'PACKAGE_PREFIX', 'EXCLUDE'): - continue # TODO configure error - - if external is not None: - continue # TODO configure error - - external = ex - - if external: - unit.onpeerdir(external) - - for k, v in data.items(): - if not v: - data.pop(k) - - dart = 'JAVA_DART: ' + base64.b64encode(json.dumps(data)) + '\n' + DELIM + '\n' - - unit.set_property(['JAVA_DART_DATA', dart]) + for java_srcs_args in data['JAVA_SRCS']: + external = None + + for i in xrange(len(java_srcs_args)): + arg = java_srcs_args[i] + + if arg == 'EXTERNAL': + if not i + 1 < len(java_srcs_args): + continue # TODO configure error + + ex = java_srcs_args[i + 1] + + if ex in ('EXTERNAL', 'SRCDIR', 'PACKAGE_PREFIX', 'EXCLUDE'): + continue # TODO configure error + + if external is not None: + continue # TODO configure error + + external = ex + + if external: + unit.onpeerdir(external) + + for k, v in data.items(): + if not v: + data.pop(k) + + dart = 'JAVA_DART: ' + base64.b64encode(json.dumps(data)) + '\n' + DELIM + '\n' + + unit.set_property(['JAVA_DART_DATA', dart]) if not idea_only and unit.get('MODULE_TYPE') in ('JAVA_PROGRAM', 'JAVA_LIBRARY', 'JTEST', 'TESTNG', 'JUNIT5') and not unit.path().startswith('$S/contrib/java'): unit.on_add_classpath_clash_check() if unit.get('LINT_LEVEL_VALUE') != "none": diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 87252f1ba0..f32a2d39a0 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -498,7 +498,7 @@ def onpy_srcs(unit, *args): unit.onpeerdir(py_runtime_path) unit.onpeerdir(unit.get("PY_PROTO_DEPS").split()) - + proto_paths = [path for path, mod in protos] unit.on_generate_py_protos_internal(proto_paths) unit.onpy_srcs([ @@ -506,7 +506,7 @@ def onpy_srcs(unit, *args): for path, mod in protos for py_suf in unit.get("PY_PROTO_SUFFIXES").split() ]) - + if optimize_proto and need_gazetteer_peerdir: unit.onpeerdir(['kernel/gazetteer/proto']) diff --git a/build/plugins/scarab_cant_clash.py b/build/plugins/scarab_cant_clash.py index 5f3dcf6c35..77dc303183 100644 --- a/build/plugins/scarab_cant_clash.py +++ b/build/plugins/scarab_cant_clash.py @@ -1,65 +1,65 @@ -import _common as common - - -def onacceleo(unit, *args): +import _common as common + + +def onacceleo(unit, *args): if unit.get("YMAKE_JAVA_MODULES") == "yes": return - flat, kv = common.sort_by_keywords( + flat, kv = common.sort_by_keywords( {'XSD': -1, 'MTL': -1, 'MTL_ROOT': 1, 'LANG': -1, 'OUT': -1, 'OUT_NOAUTO': -1, 'OUTPUT_INCLUDES': -1, 'DEBUG': 0}, - args - ) - - try: - mtlroot = kv['MTL_ROOT'][0] - except Exception: - mtlroot = unit.path().replace('$S/', '') - - classpath = ['$SCARAB', ] # XXX special word for ya make to replace following paths with real classpath - classpath.append('tools/acceleo') - + args + ) + + try: + mtlroot = kv['MTL_ROOT'][0] + except Exception: + mtlroot = unit.path().replace('$S/', '') + + classpath = ['$SCARAB', ] # XXX special word for ya make to replace following paths with real classpath + classpath.append('tools/acceleo') + depends = [] - if not unit.get('IDE_MSVS_CALL'): - for jar in classpath[1:]: + if not unit.get('IDE_MSVS_CALL'): + for jar in classpath[1:]: depends.append(jar) - - classpath = ':'.join(classpath) - - # Generate java cmd - cmd = [ - '-classpath', - classpath, - '-Dfile.encoding=UTF-8', - 'ru.yandex.se.logsng.tool.Cli', - ] - - for xsd in kv.get('XSD', []): - cmd += ['--xsd', xsd] - - for mtl in kv.get('MTL', []): - cmd += ['--mtl', mtl] - - for lang in kv.get('LANG', []): - cmd += ['--lang', lang] - - cmd += ['--output-dir', unit.path().replace('$S/', '${ARCADIA_BUILD_ROOT}/')] - cmd += ['--build-root', '${ARCADIA_BUILD_ROOT}'] - cmd += ['--source-root', '${ARCADIA_ROOT}'] - cmd += ['--mtl-root', mtlroot] - - # Generate RUN_JAVA args - run_java = cmd - - if 'DEBUG' not in kv: - run_java += ['HIDE_OUTPUT'] - - inputs = kv.get('XSD', []) + kv.get('MTL', []) + kv.get('LANG', []) - if inputs: - run_java += ['IN'] + inputs - - for k in 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES': - if kv.get(k): - run_java += [k] + kv[k] - + + classpath = ':'.join(classpath) + + # Generate java cmd + cmd = [ + '-classpath', + classpath, + '-Dfile.encoding=UTF-8', + 'ru.yandex.se.logsng.tool.Cli', + ] + + for xsd in kv.get('XSD', []): + cmd += ['--xsd', xsd] + + for mtl in kv.get('MTL', []): + cmd += ['--mtl', mtl] + + for lang in kv.get('LANG', []): + cmd += ['--lang', lang] + + cmd += ['--output-dir', unit.path().replace('$S/', '${ARCADIA_BUILD_ROOT}/')] + cmd += ['--build-root', '${ARCADIA_BUILD_ROOT}'] + cmd += ['--source-root', '${ARCADIA_ROOT}'] + cmd += ['--mtl-root', mtlroot] + + # Generate RUN_JAVA args + run_java = cmd + + if 'DEBUG' not in kv: + run_java += ['HIDE_OUTPUT'] + + inputs = kv.get('XSD', []) + kv.get('MTL', []) + kv.get('LANG', []) + if inputs: + run_java += ['IN'] + inputs + + for k in 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES': + if kv.get(k): + run_java += [k] + kv[k] + if depends: run_java += ['TOOL'] + depends diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index a490589b4c..8970837f0f 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -1,9 +1,9 @@ import os import re import sys -import json +import json import copy -import base64 +import base64 import shlex import _common import lib._metric_resolvers as mr @@ -348,7 +348,7 @@ def count_entries(x): def get_values_list(unit, key): - res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split()) + res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split()) return [r for r in res if r and r not in ['""', "''"]] @@ -416,7 +416,7 @@ def get_project_tidy_config(unit): def onadd_ytest(unit, *args): keywords = {"DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0} - flat_args, spec_args = _common.sort_by_keywords(keywords, args) + flat_args, spec_args = _common.sort_by_keywords(keywords, args) test_data = sorted(_common.filter_out_by_keyword(spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED')) @@ -547,7 +547,7 @@ def onadd_check(unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return - flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, + flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1}, args) check_type = flat_args[0] test_dir = get_norm_unit_path(unit) @@ -770,40 +770,40 @@ def add_test_to_dart(unit, test_type, binary_path=None, runner_bin=None): _dump_test(unit, test_type, test_files, timeout, get_norm_unit_path(unit), custom_deps, test_data, python_paths, split_factor, fork_mode, test_size, tags, requirements, binary_path, test_cwd=test_cwd, runner_bin=runner_bin, yt_spec=yt_spec, data_files=data_files) -def extract_java_system_properties(unit, args): - if len(args) % 2: +def extract_java_system_properties(unit, args): + if len(args) % 2: return [], 'Wrong use of SYSTEM_PROPERTIES in {}: odd number of arguments'.format(unit.path()) - + props = [] - for x, y in zip(args[::2], args[1::2]): - if x == 'FILE': - if y.startswith('${BINDIR}') or y.startswith('${ARCADIA_BUILD_ROOT}') or y.startswith('/'): + for x, y in zip(args[::2], args[1::2]): + if x == 'FILE': + if y.startswith('${BINDIR}') or y.startswith('${ARCADIA_BUILD_ROOT}') or y.startswith('/'): return [], 'Wrong use of SYSTEM_PROPERTIES in {}: absolute/build file path {}'.format(unit.path(), y) - - y = _common.rootrel_arc_src(y, unit) - if not os.path.exists(unit.resolve('$S/' + y)): + + y = _common.rootrel_arc_src(y, unit) + if not os.path.exists(unit.resolve('$S/' + y)): return [], 'Wrong use of SYSTEM_PROPERTIES in {}: can\'t resolve {}'.format(unit.path(), y) - - y = '${ARCADIA_ROOT}/' + y - props.append({'type': 'file', 'path': y}) - else: - props.append({'type': 'inline', 'key': x, 'value': y}) - + + y = '${ARCADIA_ROOT}/' + y + props.append({'type': 'file', 'path': y}) + else: + props.append({'type': 'inline', 'key': x, 'value': y}) + return props, None - - -def onjava_test(unit, *args): + + +def onjava_test(unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return - assert unit.get('MODULE_TYPE') is not None - - if unit.get('MODULE_TYPE') == 'JTEST_FOR': - if not unit.get('UNITTEST_DIR'): + assert unit.get('MODULE_TYPE') is not None + + if unit.get('MODULE_TYPE') == 'JTEST_FOR': + if not unit.get('UNITTEST_DIR'): ymake.report_configure_error('skip JTEST_FOR in {}: no args provided'.format(unit.path())) return - + java_cp_arg_type = unit.get('JAVA_CLASSPATH_CMD_TYPE_VALUE') or 'MANIFEST' if java_cp_arg_type not in ('MANIFEST', 'COMMAND_FILE', 'LIST'): ymake.report_configure_error('{}: TEST_JAVA_CLASSPATH_CMD_TYPE({}) are invalid. Choose argument from MANIFEST, COMMAND_FILE or LIST)'.format(unit.path(), java_cp_arg_type)) @@ -811,11 +811,11 @@ def onjava_test(unit, *args): unit_path = unit.path() path = _common.strip_roots(unit_path) - + test_data = get_norm_paths(unit, 'TEST_DATA_VALUE') test_data.append('arcadia/build/scripts/run_junit.py') test_data.append('arcadia/build/scripts/unpacking_jtest_runner.py') - + data, data_files = get_canonical_test_resources(unit) test_data += data @@ -823,49 +823,49 @@ def onjava_test(unit, *args): if error_mgs: ymake.report_configure_error(error_mgs) return - for prop in props: - if prop['type'] == 'file': - test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia')) - - props = base64.b64encode(json.dumps(props, encoding='utf-8')) - - test_cwd = unit.get('TEST_CWD_VALUE') or '' # TODO: validate test_cwd value - + for prop in props: + if prop['type'] == 'file': + test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia')) + + props = base64.b64encode(json.dumps(props, encoding='utf-8')) + + test_cwd = unit.get('TEST_CWD_VALUE') or '' # TODO: validate test_cwd value + if unit.get('MODULE_TYPE') == 'JUNIT5': script_rel_path = 'junit5.test' else: script_rel_path = 'junit.test' ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes' - test_record = { - 'SOURCE-FOLDER-PATH': path, - 'TEST-NAME': '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)]), + test_record = { + 'SOURCE-FOLDER-PATH': path, + 'TEST-NAME': '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)]), 'SCRIPT-REL-PATH': script_rel_path, 'TEST-TIMEOUT': unit.get('TEST_TIMEOUT') or '', 'TESTED-PROJECT-NAME': path, 'TEST-ENV': prepare_env(unit.get("TEST_ENV_VALUE")), # 'TEST-PRESERVE-ENV': 'da', 'TEST-DATA': serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED'))), - 'FORK-MODE': unit.get('TEST_FORK_MODE') or '', - 'SPLIT-FACTOR': unit.get('TEST_SPLIT_FACTOR') or '', - 'CUSTOM-DEPENDENCIES': ' '.join(get_values_list(unit, 'TEST_DEPENDS_VALUE')), + 'FORK-MODE': unit.get('TEST_FORK_MODE') or '', + 'SPLIT-FACTOR': unit.get('TEST_SPLIT_FACTOR') or '', + 'CUSTOM-DEPENDENCIES': ' '.join(get_values_list(unit, 'TEST_DEPENDS_VALUE')), 'TAG': serialize_list(_get_test_tags(unit)), - 'SIZE': unit.get('TEST_SIZE_NAME') or '', + 'SIZE': unit.get('TEST_SIZE_NAME') or '', 'REQUIREMENTS': serialize_list(get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')), 'TEST-RECIPES': prepare_recipes(unit.get("TEST_RECIPES_VALUE")), - - # JTEST/JTEST_FOR only - 'MODULE_TYPE': unit.get('MODULE_TYPE'), - 'UNITTEST_DIR': unit.get('UNITTEST_DIR') or '', - 'JVM_ARGS': serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE')), - 'SYSTEM_PROPERTIES': props, - 'TEST-CWD': test_cwd, + + # JTEST/JTEST_FOR only + 'MODULE_TYPE': unit.get('MODULE_TYPE'), + 'UNITTEST_DIR': unit.get('UNITTEST_DIR') or '', + 'JVM_ARGS': serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE')), + 'SYSTEM_PROPERTIES': props, + 'TEST-CWD': test_cwd, 'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '', 'JAVA_CLASSPATH_CMD_TYPE': java_cp_arg_type, 'NO_JBUILD': 'yes' if ymake_java_test else 'no', 'JDK_RESOURCE': 'JDK' + (unit.get('JDK_VERSION') or '_DEFAULT'), 'JDK_FOR_TESTS': 'JDK' + (unit.get('JDK_VERSION') or '_DEFAULT') + '_FOR_TESTS', - } + } test_classpath_origins = unit.get('TEST_CLASSPATH_VALUE') if test_classpath_origins: test_record['TEST_CLASSPATH_ORIGINS'] = test_classpath_origins @@ -877,12 +877,12 @@ def onjava_test(unit, *args): test_record['TEST_JAR'] = '${UNITTEST_MOD}' else: test_record['TEST_JAR'] = '{}/{}.jar'.format(unit.get('MODDIR'), unit.get('REALPRJNAME')) - + data = dump_test(unit, test_record) if data: unit.set_property(['DART_DATA', data]) - - + + def onjava_test_deps(unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests diff --git a/build/scripts/collect_java_srcs.py b/build/scripts/collect_java_srcs.py index a8056932cc..170002520a 100644 --- a/build/scripts/collect_java_srcs.py +++ b/build/scripts/collect_java_srcs.py @@ -1,51 +1,51 @@ -import os -import sys -import contextlib -import tarfile -import zipfile - - -if __name__ == '__main__': - build_root = sys.argv[1] +import os +import sys +import contextlib +import tarfile +import zipfile + + +if __name__ == '__main__': + build_root = sys.argv[1] root = os.path.normpath(sys.argv[2]) dest = os.path.normpath(sys.argv[3]) - srcs = sys.argv[4:] - - for src in srcs: + srcs = sys.argv[4:] + + for src in srcs: src = os.path.normpath(src) if src.endswith('.java') or src.endswith('.kt'): - src_rel_path = os.path.relpath(src, root) - - if os.path.join(root, src_rel_path) == src: - # Inside root - dst = os.path.join(dest, src_rel_path) - - else: - # Outside root + src_rel_path = os.path.relpath(src, root) + + if os.path.join(root, src_rel_path) == src: + # Inside root + dst = os.path.join(dest, src_rel_path) + + else: + # Outside root print>>sys.stderr, 'External src file "{}" is outside of srcdir {}, ignore'.format( - os.path.relpath(src, build_root), - os.path.relpath(root, build_root), - ) - continue - - if os.path.exists(dst): - print>>sys.stderr, 'Duplicate external src file {}, choice is undefined'.format( - os.path.relpath(dst, root) - ) - - else: + os.path.relpath(src, build_root), + os.path.relpath(root, build_root), + ) + continue + + if os.path.exists(dst): + print>>sys.stderr, 'Duplicate external src file {}, choice is undefined'.format( + os.path.relpath(dst, root) + ) + + else: destdir = os.path.dirname(dst) if destdir and not os.path.exists(destdir): os.makedirs(destdir) os.rename(src, dst) - - elif src.endswith('.jsr'): - with contextlib.closing(tarfile.open(src, 'r')) as tf: - tf.extractall(dst) - - elif src.endswith('-sources.jar'): + + elif src.endswith('.jsr'): + with contextlib.closing(tarfile.open(src, 'r')) as tf: + tf.extractall(dst) + + elif src.endswith('-sources.jar'): with zipfile.ZipFile(src) as zf: - zf.extractall(dst) - - else: - print>>sys.stderr, 'Unrecognized file type', os.path.relpath(src, build_root) + zf.extractall(dst) + + else: + print>>sys.stderr, 'Unrecognized file type', os.path.relpath(src, build_root) diff --git a/build/scripts/compile_java.py b/build/scripts/compile_java.py index 8f2ad79e4a..e95869e853 100644 --- a/build/scripts/compile_java.py +++ b/build/scripts/compile_java.py @@ -1,29 +1,29 @@ -import optparse -import contextlib -import os -import shutil -import subprocess as sp -import tarfile -import zipfile +import optparse +import contextlib +import os +import shutil +import subprocess as sp +import tarfile +import zipfile import sys - - + + def parse_args(args): - parser = optparse.OptionParser() - parser.add_option('--javac-bin') - parser.add_option('--jar-bin') + parser = optparse.OptionParser() + parser.add_option('--javac-bin') + parser.add_option('--jar-bin') parser.add_option('--vcs-mf') - parser.add_option('--package-prefix') - parser.add_option('--jar-output') - parser.add_option('--srcs-jar-output') + parser.add_option('--package-prefix') + parser.add_option('--jar-output') + parser.add_option('--srcs-jar-output') return parser.parse_args(args) - - -def mkdir_p(directory): - if not os.path.exists(directory): - os.makedirs(directory) - - + + +def mkdir_p(directory): + if not os.path.exists(directory): + os.makedirs(directory) + + def split_cmd_by_delim(cmd, delim='DELIM'): result = [[]] for arg in cmd: @@ -34,69 +34,69 @@ def split_cmd_by_delim(cmd, delim='DELIM'): return result -def main(): +def main(): cmd_parts = split_cmd_by_delim(sys.argv) assert len(cmd_parts) == 3 args, javac_opts, peers = cmd_parts opts, jsrcs = parse_args(args) - + jsrcs += list(filter(lambda x: x.endswith('.jsrc'), peers)) peers = list(filter(lambda x: not x.endswith('.jsrc'), peers)) - sources_dir = 'src' - mkdir_p(sources_dir) - for s in jsrcs: - if s.endswith('.jsrc'): - with contextlib.closing(tarfile.open(s, 'r')) as tf: - tf.extractall(sources_dir) - - srcs = [] - for r, _, files in os.walk(sources_dir): - for f in files: - srcs.append(os.path.join(r, f)) + sources_dir = 'src' + mkdir_p(sources_dir) + for s in jsrcs: + if s.endswith('.jsrc'): + with contextlib.closing(tarfile.open(s, 'r')) as tf: + tf.extractall(sources_dir) + + srcs = [] + for r, _, files in os.walk(sources_dir): + for f in files: + srcs.append(os.path.join(r, f)) srcs += jsrcs srcs = list(filter(lambda x: x.endswith('.java'), srcs)) - - classes_dir = 'cls' - mkdir_p(classes_dir) + + classes_dir = 'cls' + mkdir_p(classes_dir) classpath = os.pathsep.join(peers) - - if srcs: + + if srcs: temp_sources_file = 'temp.sources.list' with open(temp_sources_file, 'w') as ts: ts.write(' '.join(srcs)) sp.check_call([opts.javac_bin, '-nowarn', '-g', '-classpath', classpath, '-encoding', 'UTF-8', '-d', classes_dir] + javac_opts + ['@' + temp_sources_file]) - - for s in jsrcs: - if s.endswith('-sources.jar'): - with zipfile.ZipFile(s) as zf: - zf.extractall(sources_dir) - - elif s.endswith('.jar'): - with zipfile.ZipFile(s) as zf: - zf.extractall(classes_dir) - + + for s in jsrcs: + if s.endswith('-sources.jar'): + with zipfile.ZipFile(s) as zf: + zf.extractall(sources_dir) + + elif s.endswith('.jar'): + with zipfile.ZipFile(s) as zf: + zf.extractall(classes_dir) + if opts.vcs_mf: sp.check_call([opts.jar_bin, 'cfm', opts.jar_output, opts.vcs_mf, os.curdir], cwd=classes_dir) else: sp.check_call([opts.jar_bin, 'cfM', opts.jar_output, os.curdir], cwd=classes_dir) - - if opts.srcs_jar_output: - for s in jsrcs: - if s.endswith('.java'): - if opts.package_prefix: - d = os.path.join(sources_dir, *(opts.package_prefix.split('.') + [os.path.basename(s)])) - - else: - d = os.path.join(sources_dir, os.path.basename(s)) - - shutil.copyfile(s, d) - + + if opts.srcs_jar_output: + for s in jsrcs: + if s.endswith('.java'): + if opts.package_prefix: + d = os.path.join(sources_dir, *(opts.package_prefix.split('.') + [os.path.basename(s)])) + + else: + d = os.path.join(sources_dir, os.path.basename(s)) + + shutil.copyfile(s, d) + if opts.vcs_mf: sp.check_call([opts.jar_bin, 'cfm', opts.srcs_jar_output, opts.vcs_mf, os.curdir], cwd=sources_dir) else: sp.check_call([opts.jar_bin, 'cfM', opts.srcs_jar_output, os.curdir], cwd=sources_dir) - - -if __name__ == '__main__': - main() + + +if __name__ == '__main__': + main() diff --git a/build/scripts/copy_to_dir.py b/build/scripts/copy_to_dir.py index 8044b0b0bf..9baeb5ffac 100644 --- a/build/scripts/copy_to_dir.py +++ b/build/scripts/copy_to_dir.py @@ -2,19 +2,19 @@ import errno import sys import os import shutil -import optparse -import tarfile +import optparse +import tarfile -def parse_args(): - parser = optparse.OptionParser() - parser.add_option('--build-root') - parser.add_option('--dest-dir') - parser.add_option('--dest-arch') - return parser.parse_args() - - -def ensure_dir_exists(path): +def parse_args(): + parser = optparse.OptionParser() + parser.add_option('--build-root') + parser.add_option('--dest-dir') + parser.add_option('--dest-arch') + return parser.parse_args() + + +def ensure_dir_exists(path): try: os.makedirs(path) except OSError as e: @@ -24,52 +24,52 @@ def ensure_dir_exists(path): raise -def hardlink_or_copy(src, dst): - if os.name == 'nt': - shutil.copy(src, dst) - else: - try: - os.link(src, dst) - except OSError as e: - if e.errno == errno.EEXIST: - return - elif e.errno == errno.EXDEV: +def hardlink_or_copy(src, dst): + if os.name == 'nt': + shutil.copy(src, dst) + else: + try: + os.link(src, dst) + except OSError as e: + if e.errno == errno.EEXIST: + return + elif e.errno == errno.EXDEV: sys.stderr.write("Can't make cross-device hardlink - fallback to copy: {} -> {}\n".format(src, dst)) - shutil.copy(src, dst) - else: - raise - - + shutil.copy(src, dst) + else: + raise + + def main(): - opts, args = parse_args() - assert opts.build_root - assert opts.dest_dir - - dest_arch = None - if opts.dest_arch: - if opts.dest_arch.endswith('.tar'): + opts, args = parse_args() + assert opts.build_root + assert opts.dest_dir + + dest_arch = None + if opts.dest_arch: + if opts.dest_arch.endswith('.tar'): dest_arch = tarfile.open(opts.dest_arch, 'w', dereference=True) - elif opts.dest_arch.endswith('.tar.gz') or opts.dest_arch.endswith('.tgz'): + elif opts.dest_arch.endswith('.tar.gz') or opts.dest_arch.endswith('.tgz'): dest_arch = tarfile.open(opts.dest_arch, 'w:gz', dereference=True) - else: - # TODO: move check to graph generation stage - raise Exception('Unsopported archive type for {}. Use one of: tar, tar.gz, tgz.'.format(os.path.basename(opts.dest_arch))) - - for arg in args: + else: + # TODO: move check to graph generation stage + raise Exception('Unsopported archive type for {}. Use one of: tar, tar.gz, tgz.'.format(os.path.basename(opts.dest_arch))) + + for arg in args: dst = arg - if dst.startswith(opts.build_root): - dst = dst[len(opts.build_root) + 1:] + if dst.startswith(opts.build_root): + dst = dst[len(opts.build_root) + 1:] if dest_arch and not arg.endswith('.pkg.fake'): - dest_arch.add(arg, arcname=dst) + dest_arch.add(arg, arcname=dst) + + dst = os.path.join(opts.dest_dir, dst) + ensure_dir_exists(os.path.dirname(dst)) + hardlink_or_copy(arg, dst) + + if dest_arch: + dest_arch.close() + - dst = os.path.join(opts.dest_dir, dst) - ensure_dir_exists(os.path.dirname(dst)) - hardlink_or_copy(arg, dst) - - if dest_arch: - dest_arch.close() - - if __name__ == '__main__': sys.exit(main()) diff --git a/build/scripts/create_jcoverage_report.py b/build/scripts/create_jcoverage_report.py index 578183e9b9..45083ff4f7 100644 --- a/build/scripts/create_jcoverage_report.py +++ b/build/scripts/create_jcoverage_report.py @@ -1,8 +1,8 @@ import argparse import tarfile -import zipfile +import zipfile import os -import sys +import sys import time import subprocess @@ -26,66 +26,66 @@ class Timer(object): def main(source, output, java, prefix_filter, exclude_filter, jars_list, output_format, tar_output, agent_disposition, runners_paths): timer = Timer() - reports_dir = 'jacoco_reports_dir' - mkdir_p(reports_dir) - with tarfile.open(source) as tf: - tf.extractall(reports_dir) + reports_dir = 'jacoco_reports_dir' + mkdir_p(reports_dir) + with tarfile.open(source) as tf: + tf.extractall(reports_dir) timer.step("Coverage data extracted") - reports = [os.path.join(reports_dir, fname) for fname in os.listdir(reports_dir)] - - with open(jars_list) as f: - jars = f.read().strip().split() + reports = [os.path.join(reports_dir, fname) for fname in os.listdir(reports_dir)] + + with open(jars_list) as f: + jars = f.read().strip().split() if jars and runners_paths: for r in runners_paths: try: jars.remove(r) except ValueError: pass - - src_dir = 'sources_dir' - cls_dir = 'classes_dir' - - mkdir_p(src_dir) - mkdir_p(cls_dir) - - for jar in jars: - if jar.endswith('devtools-jacoco-agent.jar'): - agent_disposition = jar - + + src_dir = 'sources_dir' + cls_dir = 'classes_dir' + + mkdir_p(src_dir) + mkdir_p(cls_dir) + + for jar in jars: + if jar.endswith('devtools-jacoco-agent.jar'): + agent_disposition = jar + # Skip java contrib - it's irrelevant coverage if jar.startswith('contrib/java'): continue - with zipfile.ZipFile(jar) as jf: - for entry in jf.infolist(): - if entry.filename.endswith('.java'): - dest = src_dir - - elif entry.filename.endswith('.class'): - dest = cls_dir - - else: - continue - + with zipfile.ZipFile(jar) as jf: + for entry in jf.infolist(): + if entry.filename.endswith('.java'): + dest = src_dir + + elif entry.filename.endswith('.class'): + dest = cls_dir + + else: + continue + entry.filename = entry.filename.encode('utf-8') - jf.extract(entry, dest) + jf.extract(entry, dest) timer.step("Jar files extracted") - - if not agent_disposition: - print>>sys.stderr, 'Can\'t find jacoco agent. Will not generate html report for java coverage.' - + + if not agent_disposition: + print>>sys.stderr, 'Can\'t find jacoco agent. Will not generate html report for java coverage.' + if tar_output: report_dir = 'java.report.temp' else: report_dir = output mkdir_p(report_dir) - if agent_disposition: + if agent_disposition: agent_cmd = [java, '-jar', agent_disposition, src_dir, cls_dir, prefix_filter or '.', exclude_filter or '__no_exclude__', report_dir, output_format] - agent_cmd += reports - subprocess.check_call(agent_cmd) + agent_cmd += reports + subprocess.check_call(agent_cmd) timer.step("Jacoco finished") - + if tar_output: with tarfile.open(output, 'w') as outf: outf.add(report_dir, arcname='.') @@ -103,7 +103,7 @@ if __name__ == '__main__': parser.add_argument('--java', action='store') parser.add_argument('--prefix-filter', action='store') parser.add_argument('--exclude-filter', action='store') - parser.add_argument('--jars-list', action='store') + parser.add_argument('--jars-list', action='store') parser.add_argument('--output-format', action='store', default="html") parser.add_argument('--raw-output', dest='tar_output', action='store_false', default=True) parser.add_argument('--agent-disposition', action='store') diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py index 3860974ac4..db4fea50bf 100755 --- a/build/scripts/fetch_from.py +++ b/build/scripts/fetch_from.py @@ -321,15 +321,15 @@ def process(fetched_file, file_name, args, remove=True): hardlink_or_copy(fetched_file, args.copy_to) if not args.outputs: args.outputs = [args.copy_to] - + if args.rename_to: args.rename.append(fetched_file) if not args.outputs: args.outputs = [args.rename_to] - + if args.copy_to_dir: hardlink_or_copy(fetched_file, os.path.join(args.copy_to_dir, file_name)) - + if args.untar_to: ensure_dir(args.untar_to) # Extract only requested files diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py index f8533f1223..a99542e174 100755 --- a/build/scripts/fetch_from_sandbox.py +++ b/build/scripts/fetch_from_sandbox.py @@ -18,19 +18,19 @@ MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' TEMPORARY_ERROR_CODES = (429, 500, 503, 504) -def parse_args(): +def parse_args(): parser = argparse.ArgumentParser() fetch_from.add_common_arguments(parser) parser.add_argument('--resource-id', type=int, required=True) parser.add_argument('--custom-fetcher') parser.add_argument('--resource-file') return parser.parse_args() - - -class ResourceInfoError(Exception): - pass - - + + +class ResourceInfoError(Exception): + pass + + class UnsupportedProtocolException(Exception): pass @@ -75,37 +75,37 @@ def download_by_skynet(resource_info, file_name): def _urlopen(url, data=None, headers=None): - n = 10 + n = 10 tout = 30 started = time.time() reqid = uuid.uuid4() - + request = urllib2.Request(url, data=data, headers=headers or {}) request.add_header('X-Request-Timeout', str(tout)) request.add_header('X-Request-Id', str(reqid)) request.add_header('User-Agent', 'fetch_from_sandbox.py') - for i in xrange(n): + for i in xrange(n): retry_after = i - try: + try: request.add_header('X-Request-Duration', str(int(time.time() - started))) return urllib2.urlopen(request, timeout=tout).read() - - except urllib2.HTTPError as e: + + except urllib2.HTTPError as e: logging.warning('failed to fetch URL %s with HTTP code %d: %s', url, e.code, e) retry_after = int(e.headers.get('Retry-After', str(retry_after))) - + if e.code not in TEMPORARY_ERROR_CODES: - raise - - except Exception as e: + raise + + except Exception as e: logging.warning('failed to fetch URL %s: %s', url, e) - - if i + 1 == n: - raise e - + + if i + 1 == n: + raise e + time.sleep(retry_after) - - + + def _query(url): return json.loads(_urlopen(url)) @@ -133,19 +133,19 @@ def fetch_via_script(script, resource_id): def fetch(resource_id, custom_fetcher): - try: + try: resource_info = get_resource_info(resource_id, touch=True, no_links=True) - except Exception as e: + except Exception as e: sys.stderr.write( "Failed to fetch resource {}: {}\n".format(resource_id, str(e)) ) raise - + if resource_info.get('state', 'DELETED') != 'READY': raise ResourceInfoError("Resource {} is not READY".format(resource_id)) - logging.info('Resource %s info %s', str(resource_id), json.dumps(resource_info)) - + logging.info('Resource %s info %s', str(resource_id), json.dumps(resource_info)) + resource_file_name = os.path.basename(resource_info["file_name"]) expected_md5 = resource_info.get('md5') @@ -183,9 +183,9 @@ def fetch(resource_id, custom_fetcher): if mds_link is not None: yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5) - if resource_info.get('attributes', {}).get('ttl') != 'inf': - sys.stderr.write('WARNING: resource {} ttl is not "inf".\n'.format(resource_id)) - + if resource_info.get('attributes', {}).get('ttl') != 'inf': + sys.stderr.write('WARNING: resource {} ttl is not "inf".\n'.format(resource_id)) + exc_info = None for i, action in enumerate(itertools.islice(iter_tries(), 0, 10)): try: @@ -210,10 +210,10 @@ def fetch(resource_id, custom_fetcher): raise exc_info[0], exc_info[1], exc_info[2] else: raise Exception("No available protocol and/or server to fetch resource") - + return fetched_file, resource_info['file_name'] - + def _get_resource_info_from_file(resource_file): if resource_file is None or not os.path.exists(resource_file): return None @@ -242,7 +242,7 @@ def _get_resource_info_from_file(resource_file): def main(args): custom_fetcher = os.environ.get('YA_CUSTOM_FETCHER') - + resource_info = _get_resource_info_from_file(args.resource_file) if resource_info: fetched_file = args.resource_file @@ -250,14 +250,14 @@ def main(args): else: # This code should be merged to ya and removed. fetched_file, file_name = fetch(args.resource_id, custom_fetcher) - + fetch_from.process(fetched_file, file_name, args, remove=not custom_fetcher and not resource_info) if __name__ == '__main__': args = parse_args() fetch_from.setup_logging(args, os.path.basename(__file__)) - + try: main(args) except Exception as e: diff --git a/build/scripts/fetch_resource.py b/build/scripts/fetch_resource.py index ba046ad88e..d5af311e5d 100644 --- a/build/scripts/fetch_resource.py +++ b/build/scripts/fetch_resource.py @@ -1,43 +1,43 @@ -import urllib2 +import urllib2 import argparse -import xmlrpclib - - -def parse_args(): +import xmlrpclib + + +def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-r', '--resource-id', type=int, required=True) parser.add_argument('-o', '--output', required=True) return parser.parse_args() - - -def fetch(url, retries=4, timeout=5): - for i in xrange(retries): - try: - return urllib2.urlopen(url, timeout=timeout).read() - - except Exception: - if i + 1 < retries: - continue - - else: - raise - - -def fetch_resource(id_): + + +def fetch(url, retries=4, timeout=5): + for i in xrange(retries): + try: + return urllib2.urlopen(url, timeout=timeout).read() + + except Exception: + if i + 1 < retries: + continue + + else: + raise + + +def fetch_resource(id_): urls = xmlrpclib.ServerProxy("https://sandbox.yandex-team.ru/sandbox/xmlrpc").get_resource_http_links(id_) - - for u in urls: - try: - return fetch(u) - + + for u in urls: + try: + return fetch(u) + except Exception: - continue - - raise Exception('Cannot fetch resource {}'.format(id_)) - - -if __name__ == '__main__': + continue + + raise Exception('Cannot fetch resource {}'.format(id_)) + + +if __name__ == '__main__': args = parse_args() - + with open(args.output, 'wb') as f: f.write(fetch_resource(int(args.resource_id))) diff --git a/build/scripts/generate_pom.py b/build/scripts/generate_pom.py index 28d5c988eb..200caebc0b 100644 --- a/build/scripts/generate_pom.py +++ b/build/scripts/generate_pom.py @@ -247,7 +247,7 @@ def build_pom_and_export_to_maven(**kwargs): _indent(project) - et.ElementTree(project).write(pom_path) + et.ElementTree(project).write(pom_path) sys.stderr.write("[MAVEN EXPORT] Generated {} file for target {}\n".format(os.path.basename(pom_path), target_path)) diff --git a/build/scripts/link_dyn_lib.py b/build/scripts/link_dyn_lib.py index 0b14f0cc99..23487f5c1e 100644 --- a/build/scripts/link_dyn_lib.py +++ b/build/scripts/link_dyn_lib.py @@ -3,7 +3,7 @@ import os import subprocess import tempfile import collections -import optparse +import optparse import pipes from process_whole_archive_option import ProcessWholeArchiveOption @@ -158,26 +158,26 @@ def fix_cmd(arch, musl, c): return sum((do_fix(x) for x in c), []) -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--arch') - parser.add_option('--target') - parser.add_option('--soname') - parser.add_option('--fix-elf') +def parse_args(): + parser = optparse.OptionParser() + parser.disable_interspersed_args() + parser.add_option('--arch') + parser.add_option('--target') + parser.add_option('--soname') + parser.add_option('--fix-elf') parser.add_option('--linker-output') parser.add_option('--musl', action='store_true') parser.add_option('--whole-archive-peers', action='append') parser.add_option('--whole-archive-libs', action='append') - return parser.parse_args() + return parser.parse_args() -if __name__ == '__main__': - opts, args = parse_args() - - assert opts.arch - assert opts.target - +if __name__ == '__main__': + opts, args = parse_args() + + assert opts.arch + assert opts.target + cmd = fix_cmd(opts.arch, opts.musl, args) cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd) @@ -187,27 +187,27 @@ if __name__ == '__main__': stdout = sys.stdout proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=stdout) - proc.communicate() - - if proc.returncode: + proc.communicate() + + if proc.returncode: print >>sys.stderr, 'linker has failed with retcode:', proc.returncode print >>sys.stderr, 'linker command:', shlex_join(cmd) - sys.exit(proc.returncode) - - if opts.fix_elf: - cmd = [opts.fix_elf, opts.target] - proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=sys.stdout) - proc.communicate() - - if proc.returncode: + sys.exit(proc.returncode) + + if opts.fix_elf: + cmd = [opts.fix_elf, opts.target] + proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=sys.stdout) + proc.communicate() + + if proc.returncode: print >>sys.stderr, 'fix_elf has failed with retcode:', proc.returncode print >>sys.stderr, 'fix_elf command:', shlex_join(cmd) - sys.exit(proc.returncode) - - if opts.soname and opts.soname != opts.target: + sys.exit(proc.returncode) + + if opts.soname and opts.soname != opts.target: if os.path.exists(opts.soname): os.unlink(opts.soname) - os.link(opts.target, opts.soname) + os.link(opts.target, opts.soname) # -----------------Test---------------- # diff --git a/build/scripts/merge_coverage_data.py b/build/scripts/merge_coverage_data.py index 6e4aa9fde9..b7fa3c6a86 100644 --- a/build/scripts/merge_coverage_data.py +++ b/build/scripts/merge_coverage_data.py @@ -2,29 +2,29 @@ import sys import tarfile import copy import os -import uuid +import uuid def main(args): - output_file, args = args[0], args[1:] + output_file, args = args[0], args[1:] # heretic@: Splits files on which could be merged( files ) and which should not be merged( expendables ) # expendables will be in output_file in form {name}{ordinal number of archive in args[]}.{extension} - try: - split_i = args.index('-no-merge') - except ValueError: - split_i = len(args) - files, expendables = args[:split_i], args[split_i + 1:] + try: + split_i = args.index('-no-merge') + except ValueError: + split_i = len(args) + files, expendables = args[:split_i], args[split_i + 1:] with tarfile.open(output_file, 'w') as outf: - for x in files: + for x in files: with tarfile.open(x) as tf: for tarinfo in tf: new_tarinfo = copy.deepcopy(tarinfo) if new_tarinfo.name in expendables: - dirname, basename = os.path.split(new_tarinfo.name) - basename_parts = basename.split('.', 1) - new_basename = '.'.join([basename_parts[0] + str(uuid.uuid4())] + basename_parts[1:]) - new_tarinfo.name = os.path.join(dirname, new_basename) + dirname, basename = os.path.split(new_tarinfo.name) + basename_parts = basename.split('.', 1) + new_basename = '.'.join([basename_parts[0] + str(uuid.uuid4())] + basename_parts[1:]) + new_tarinfo.name = os.path.join(dirname, new_basename) outf.addfile(new_tarinfo, tf.extractfile(tarinfo)) diff --git a/build/scripts/pack_jcoverage_resources.py b/build/scripts/pack_jcoverage_resources.py index 22d2dddbe7..f6e181067a 100644 --- a/build/scripts/pack_jcoverage_resources.py +++ b/build/scripts/pack_jcoverage_resources.py @@ -9,16 +9,16 @@ def main(args): report_file = args[1] res = subprocess.call(args[args.index('-end') + 1:]) - - if not os.path.exists(report_file): - print>>sys.stderr, 'Can\'t find jacoco exec file' - return res - + + if not os.path.exists(report_file): + print>>sys.stderr, 'Can\'t find jacoco exec file' + return res + with tarfile.open(output_file, 'w') as outf: outf.add(report_file, arcname=os.path.basename(report_file)) - + return res if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/build/scripts/resolve_java_srcs.py b/build/scripts/resolve_java_srcs.py index 4919e16ed7..a2e6c20012 100644 --- a/build/scripts/resolve_java_srcs.py +++ b/build/scripts/resolve_java_srcs.py @@ -14,8 +14,8 @@ def list_all_files(directory, prefix='/', hidden_files=False): return result -def pattern_to_regexp(p): - return '^' + \ +def pattern_to_regexp(p): + return '^' + \ ('/' if not p.startswith('**') else '') + \ re.escape(p).replace( r'\*\*\/', '[_DIR_]' @@ -26,8 +26,8 @@ def pattern_to_regexp(p): ).replace( '[_FILE_]', '([^/]*)' ) + '$' - - + + def resolve_java_srcs(srcdir, include_patterns, exclude_patterns, all_resources, resolve_kotlin=False, resolve_groovy=False): result = {'java': [], 'not_java': [], 'kotlin': [], 'groovy': []} include_patterns_normal, include_patterns_hidden, exclude_patterns_normal, exclude_patterns_hidden = [], [], [], [] @@ -43,22 +43,22 @@ def resolve_java_srcs(srcdir, include_patterns, exclude_patterns, all_resources, else: re_patterns = [re.compile(i) for i in re_patterns] vis[:], hid[:] = re_patterns[:len(vis)], re_patterns[len(vis):] - + for inc_patterns, exc_patterns, with_hidden_files in ( (include_patterns_normal, exclude_patterns_normal, False), (include_patterns_hidden, exclude_patterns_hidden, True), ): for f in list_all_files(srcdir, hidden_files=with_hidden_files): excluded = False - + for exc_re in exc_patterns: if exc_re.match(f): excluded = True break - + if excluded: continue - + for inc_re in inc_patterns: if inc_re.match(f): s = os.path.normpath(f[1:]) @@ -73,7 +73,7 @@ def resolve_java_srcs(srcdir, include_patterns, exclude_patterns, all_resources, else: result['not_java'].append(s) break - + return sorted(result['java']), sorted(result['not_java']), sorted(result['kotlin']), sorted(result['groovy']) @@ -99,8 +99,8 @@ if __name__ == '__main__': parser.add_argument('--all-resources', action='store_true', default=False) parser.add_argument('--resolve-kotlin', action='store_true', default=False) parser.add_argument('--resolve-groovy', action='store_true', default=False) - parser.add_argument('--include-patterns', nargs='*', default=[]) - parser.add_argument('--exclude-patterns', nargs='*', default=[]) + parser.add_argument('--include-patterns', nargs='*', default=[]) + parser.add_argument('--exclude-patterns', nargs='*', default=[]) args = parser.parse_args() do_it(**vars(args)) diff --git a/build/scripts/run_javac.py b/build/scripts/run_javac.py index 2a870af771..c35546e0fe 100644 --- a/build/scripts/run_javac.py +++ b/build/scripts/run_javac.py @@ -1,47 +1,47 @@ -import sys -import subprocess -import optparse -import re - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--sources-list') - parser.add_option('--verbose', default=False, action='store_true') - parser.add_option('--remove-notes', default=False, action='store_true') +import sys +import subprocess +import optparse +import re + + +def parse_args(): + parser = optparse.OptionParser() + parser.disable_interspersed_args() + parser.add_option('--sources-list') + parser.add_option('--verbose', default=False, action='store_true') + parser.add_option('--remove-notes', default=False, action='store_true') parser.add_option('--ignore-errors', default=False, action='store_true') parser.add_option('--kotlin', default=False, action='store_true') - return parser.parse_args() - - -COLORING = { - r'^(?P<path>.*):(?P<line>\d*): error: (?P<msg>.*)': lambda m: '[[unimp]]{path}[[rst]]:[[alt2]]{line}[[rst]]: [[c:light-red]]error[[rst]]: [[bad]]{msg}[[rst]]'.format( - path=m.group('path'), - line=m.group('line'), - msg=m.group('msg'), - ), - r'^(?P<path>.*):(?P<line>\d*): warning: (?P<msg>.*)': lambda m: '[[unimp]]{path}[[rst]]:[[alt2]]{line}[[rst]]: [[c:light-yellow]]warning[[rst]]: {msg}'.format( - path=m.group('path'), - line=m.group('line'), - msg=m.group('msg'), - ), - r'^warning: ': lambda m: '[[c:light-yellow]]warning[[rst]]: ', - r'^error: (?P<msg>.*)': lambda m: '[[c:light-red]]error[[rst]]: [[bad]]{msg}[[rst]]'.format(msg=m.group('msg')), - r'^Note: ': lambda m: '[[c:light-cyan]]Note[[rst]]: ', -} - - -def colorize(err): - for regex, sub in COLORING.iteritems(): - err = re.sub(regex, sub, err, flags=re.MULTILINE) - return err - - -def remove_notes(err): - return '\n'.join([line for line in err.split('\n') if not line.startswith('Note:')]) - - + return parser.parse_args() + + +COLORING = { + r'^(?P<path>.*):(?P<line>\d*): error: (?P<msg>.*)': lambda m: '[[unimp]]{path}[[rst]]:[[alt2]]{line}[[rst]]: [[c:light-red]]error[[rst]]: [[bad]]{msg}[[rst]]'.format( + path=m.group('path'), + line=m.group('line'), + msg=m.group('msg'), + ), + r'^(?P<path>.*):(?P<line>\d*): warning: (?P<msg>.*)': lambda m: '[[unimp]]{path}[[rst]]:[[alt2]]{line}[[rst]]: [[c:light-yellow]]warning[[rst]]: {msg}'.format( + path=m.group('path'), + line=m.group('line'), + msg=m.group('msg'), + ), + r'^warning: ': lambda m: '[[c:light-yellow]]warning[[rst]]: ', + r'^error: (?P<msg>.*)': lambda m: '[[c:light-red]]error[[rst]]: [[bad]]{msg}[[rst]]'.format(msg=m.group('msg')), + r'^Note: ': lambda m: '[[c:light-cyan]]Note[[rst]]: ', +} + + +def colorize(err): + for regex, sub in COLORING.iteritems(): + err = re.sub(regex, sub, err, flags=re.MULTILINE) + return err + + +def remove_notes(err): + return '\n'.join([line for line in err.split('\n') if not line.startswith('Note:')]) + + def find_javac(cmd): if not cmd: return None @@ -84,39 +84,39 @@ def fix_cmd(cmd): return cmd -def main(): - opts, cmd = parse_args() - - with open(opts.sources_list) as f: - input_files = f.read().strip().split() - +def main(): + opts, cmd = parse_args() + + with open(opts.sources_list) as f: + input_files = f.read().strip().split() + if opts.kotlin: input_files = [i for i in input_files if i.endswith('.kt')] - if not input_files: - if opts.verbose: - sys.stderr.write('No files to compile, javac is not launched.\n') - - else: + if not input_files: + if opts.verbose: + sys.stderr.write('No files to compile, javac is not launched.\n') + + else: p = subprocess.Popen(fix_cmd(cmd), stderr=subprocess.PIPE) - _, err = p.communicate() - rc = p.wait() - - if opts.remove_notes: - err = remove_notes(err) - - try: - err = colorize(err) - - except Exception: - pass - + _, err = p.communicate() + rc = p.wait() + + if opts.remove_notes: + err = remove_notes(err) + + try: + err = colorize(err) + + except Exception: + pass + if opts.ignore_errors and rc: sys.stderr.write('error: javac actually failed with exit code {}\n'.format(rc)) rc = 0 - sys.stderr.write(err) - sys.exit(rc) - - -if __name__ == '__main__': - main() + sys.stderr.write(err) + sys.exit(rc) + + +if __name__ == '__main__': + main() diff --git a/build/scripts/run_sonar.py b/build/scripts/run_sonar.py index a1d06d2c41..761cc34b78 100644 --- a/build/scripts/run_sonar.py +++ b/build/scripts/run_sonar.py @@ -1,38 +1,38 @@ -import os -import sys -import zipfile -import tarfile -import subprocess as sp -import optparse -import shutil +import os +import sys +import zipfile +import tarfile +import subprocess as sp +import optparse +import shutil import xml.etree.ElementTree as et - - -def parse_args(): - parser = optparse.OptionParser() - parser.add_option( - '--classes-jar-path', - dest='classes_jar_paths', - action='append', - default=[], - ) - parser.add_option('--sources-jar-path') - parser.add_option('--sonar-scanner-jar-path') - parser.add_option('--sonar-scanner-main-class') - parser.add_option('--java-coverage-merged-tar') - parser.add_option('--java-binary-path') - parser.add_option('--log-path') + + +def parse_args(): + parser = optparse.OptionParser() + parser.add_option( + '--classes-jar-path', + dest='classes_jar_paths', + action='append', + default=[], + ) + parser.add_option('--sources-jar-path') + parser.add_option('--sonar-scanner-jar-path') + parser.add_option('--sonar-scanner-main-class') + parser.add_option('--java-coverage-merged-tar') + parser.add_option('--java-binary-path') + parser.add_option('--log-path') parser.add_option('--gcov-report-path') parser.add_option('--source-root') parser.add_option('--java-args', action='append', default=[]) - return parser.parse_args() - - -def extract_zip_file(zip_file_path, dest_dir): - with zipfile.ZipFile(zip_file_path) as arch: - arch.extractall(dest_dir) - - + return parser.parse_args() + + +def extract_zip_file(zip_file_path, dest_dir): + with zipfile.ZipFile(zip_file_path) as arch: + arch.extractall(dest_dir) + + def get_source_real_path(source_root, path): parts = os.path.normpath(path).split(os.path.sep) for i in xrange(len(parts)): @@ -64,58 +64,58 @@ def collect_cpp_sources(report, source_root, destination): os.link(src, dst) -def main(opts, props_args): - sources_dir = os.path.abspath('src') +def main(opts, props_args): + sources_dir = os.path.abspath('src') base_props_args = ['-Dsonar.sources=' + sources_dir] - os.mkdir(sources_dir) + os.mkdir(sources_dir) if opts.sources_jar_path: extract_zip_file(opts.sources_jar_path, sources_dir) if opts.gcov_report_path: collect_cpp_sources(opts.gcov_report_path, opts.source_root, sources_dir) base_props_args += ['-Dsonar.projectBaseDir=' + sources_dir, '-Dsonar.cxx.coverage.reportPath=' + opts.gcov_report_path] - - if opts.classes_jar_paths: - classes_dir = os.path.abspath('cls') - os.mkdir(classes_dir) - - for classes_jar_path in opts.classes_jar_paths: - extract_zip_file(classes_jar_path, classes_dir) - - base_props_args.append('-Dsonar.java.binaries=' + classes_dir) - - if opts.java_coverage_merged_tar: - jacoco_report_path = os.path.abspath('jacoco.exec') - with open(jacoco_report_path, 'w') as dest: - with tarfile.open(opts.java_coverage_merged_tar) as tar: - for src in tar: + + if opts.classes_jar_paths: + classes_dir = os.path.abspath('cls') + os.mkdir(classes_dir) + + for classes_jar_path in opts.classes_jar_paths: + extract_zip_file(classes_jar_path, classes_dir) + + base_props_args.append('-Dsonar.java.binaries=' + classes_dir) + + if opts.java_coverage_merged_tar: + jacoco_report_path = os.path.abspath('jacoco.exec') + with open(jacoco_report_path, 'w') as dest: + with tarfile.open(opts.java_coverage_merged_tar) as tar: + for src in tar: extracted = tar.extractfile(src) if extracted is not None: shutil.copyfileobj(extracted, dest) - - base_props_args += [ - '-Dsonar.core.codeCoveragePlugin=jacoco', - '-Dsonar.jacoco.reportPath=' + jacoco_report_path - ] + + base_props_args += [ + '-Dsonar.core.codeCoveragePlugin=jacoco', + '-Dsonar.jacoco.reportPath=' + jacoco_report_path + ] java_args = ['-{}'.format(i) for i in opts.java_args] + ['-Djava.net.preferIPv6Addresses=true', '-Djava.net.preferIPv4Addresses=false'] - - sonar_cmd = [ - opts.java_binary_path, + + sonar_cmd = [ + opts.java_binary_path, ] + java_args + [ - '-classpath', - opts.sonar_scanner_jar_path, + '-classpath', + opts.sonar_scanner_jar_path, ] + base_props_args + props_args + [opts.sonar_scanner_main_class, '-X'] - - p = sp.Popen(sonar_cmd, stdout=sp.PIPE, stderr=sp.STDOUT) - out, _ = p.communicate() - - sys.stderr.write(out) + + p = sp.Popen(sonar_cmd, stdout=sp.PIPE, stderr=sp.STDOUT) + out, _ = p.communicate() + + sys.stderr.write(out) with open(opts.log_path, 'a') as f: - f.write(out) - - sys.exit(p.returncode) - - -if __name__ == '__main__': - opts, args = parse_args() - props_args = ['-D' + arg for arg in args] - main(opts, props_args) + f.write(out) + + sys.exit(p.returncode) + + +if __name__ == '__main__': + opts, args = parse_args() + props_args = ['-D' + arg for arg in args] + main(opts, props_args) diff --git a/build/scripts/stderr2stdout.py b/build/scripts/stderr2stdout.py index f47803a450..0e510da373 100644 --- a/build/scripts/stderr2stdout.py +++ b/build/scripts/stderr2stdout.py @@ -1,6 +1,6 @@ -import subprocess -import sys - -if __name__ == '__main__': - assert len(sys.argv) > 1 - sys.exit(subprocess.Popen(sys.argv[1:], stderr=sys.stdout).wait()) +import subprocess +import sys + +if __name__ == '__main__': + assert len(sys.argv) > 1 + sys.exit(subprocess.Popen(sys.argv[1:], stderr=sys.stdout).wait()) diff --git a/build/scripts/tared_protoc.py b/build/scripts/tared_protoc.py index 3207b54b1d..7643e1dbfe 100644 --- a/build/scripts/tared_protoc.py +++ b/build/scripts/tared_protoc.py @@ -1,31 +1,31 @@ -import os -import optparse -import tarfile -import contextlib -import subprocess as sp - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() - parser.add_option('--tar-output') - parser.add_option('--protoc-out-dir') - return parser.parse_args() - - -def main(): - opts, args = parse_args() - assert opts.tar_output - assert opts.protoc_out_dir - - if not os.path.exists(opts.protoc_out_dir): - os.makedirs(opts.protoc_out_dir) - - sp.check_call(args) - - with contextlib.closing(tarfile.open(opts.tar_output, 'w')) as tf: - tf.add(opts.protoc_out_dir, arcname='') - - -if __name__ == '__main__': - main() +import os +import optparse +import tarfile +import contextlib +import subprocess as sp + + +def parse_args(): + parser = optparse.OptionParser() + parser.disable_interspersed_args() + parser.add_option('--tar-output') + parser.add_option('--protoc-out-dir') + return parser.parse_args() + + +def main(): + opts, args = parse_args() + assert opts.tar_output + assert opts.protoc_out_dir + + if not os.path.exists(opts.protoc_out_dir): + os.makedirs(opts.protoc_out_dir) + + sp.check_call(args) + + with contextlib.closing(tarfile.open(opts.tar_output, 'w')) as tf: + tf.add(opts.protoc_out_dir, arcname='') + + +if __name__ == '__main__': + main() diff --git a/build/scripts/unpacking_jtest_runner.py b/build/scripts/unpacking_jtest_runner.py index 8b23e3ed26..9730dcd711 100644 --- a/build/scripts/unpacking_jtest_runner.py +++ b/build/scripts/unpacking_jtest_runner.py @@ -3,25 +3,25 @@ import json import optparse import os import sys -import subprocess +import subprocess import time import zipfile import platform - -# This script changes test run classpath by unpacking tests.jar -> tests-dir. The goal -# is to launch tests with the same classpath as maven does. - - -def parse_args(): - parser = optparse.OptionParser() - parser.disable_interspersed_args() + +# This script changes test run classpath by unpacking tests.jar -> tests-dir. The goal +# is to launch tests with the same classpath as maven does. + + +def parse_args(): + parser = optparse.OptionParser() + parser.disable_interspersed_args() parser.add_option('--trace-file') - parser.add_option('--jar-binary') - parser.add_option('--tests-jar-path') + parser.add_option('--jar-binary') + parser.add_option('--tests-jar-path') parser.add_option('--classpath-option-type', choices=('manifest', 'command_file', 'list'), default='manifest') - return parser.parse_args() - - + return parser.parse_args() + + # temporary, for jdk8/jdk9+ compatibility def fix_cmd(cmd): if not cmd: @@ -88,18 +88,18 @@ def make_command_file_from_cp(class_path, out): cp_file.write(os.pathsep.join(class_path)) -def main(): +def main(): s = time.time() - opts, args = parse_args() - - # unpack tests jar - try: + opts, args = parse_args() + + # unpack tests jar + try: build_root = args[args.index('--build-root') + 1] dest = os.path.join(build_root, 'test-classes') - except Exception: + except Exception: build_root = '' - dest = os.path.abspath('test-classes') - + dest = os.path.abspath('test-classes') + extract_jars(dest, opts.tests_jar_path) metrics = { @@ -107,7 +107,7 @@ def main(): } s = time.time() - # fix java classpath + # fix java classpath cp_idx = args.index('-classpath') if args[cp_idx + 1].startswith('@'): real_name = args[cp_idx + 1][1:] @@ -137,12 +137,12 @@ def main(): if opts.trace_file: dump_chunk_event({'metrics': metrics}, opts.trace_file) - # run java cmd + # run java cmd if platform.system() == 'Windows': sys.exit(subprocess.Popen(args).wait()) else: os.execv(args[0], args) - - -if __name__ == '__main__': - main() + + +if __name__ == '__main__': + main() diff --git a/build/scripts/writer.py b/build/scripts/writer.py index 7eceef9ccd..21bb3006e5 100644 --- a/build/scripts/writer.py +++ b/build/scripts/writer.py @@ -1,10 +1,10 @@ import sys import argparse - + import process_command_files as pcf - -def parse_args(): + +def parse_args(): args = pcf.get_args(sys.argv[1:]) parser = argparse.ArgumentParser() parser.add_argument('-f', '--file', dest='file_path') @@ -15,7 +15,7 @@ def parse_args(): parser.add_argument('-m', '--content-multiple', nargs='*', dest='content') parser.add_argument('-P', '--path-list', action='store_true', default=False) return parser.parse_args(args) - + def smart_shell_quote(v): if v is None: @@ -23,11 +23,11 @@ def smart_shell_quote(v): if ' ' in v or '"' in v or "'" in v: return "\"{0}\"".format(v.replace('"', '\\"')) return v - -if __name__ == '__main__': + +if __name__ == '__main__': args = parse_args() open_type = 'a' if args.append else 'w' - + content = args.content if args.quote: content = [smart_shell_quote(ln) for ln in content] if content is not None else None diff --git a/build/sysincl/darwin.yml b/build/sysincl/darwin.yml index 04a6cc2ce6..582c7e1202 100644 --- a/build/sysincl/darwin.yml +++ b/build/sysincl/darwin.yml @@ -99,7 +99,7 @@ - net/if_dl.h - net/if_media.h - net/if_types.h - - net/if_var.h + - net/if_var.h - netinet/in_pcb.h - netinet/ip_var.h - netinet/tcp_fsm.h @@ -115,7 +115,7 @@ - os/proc.h - os/trace.h - panel.h - - pthread_spis.h + - pthread_spis.h - readline/history.h - readline/readline.h - readpassphrase.h @@ -179,7 +179,7 @@ - CoreFoundation/CFPreferences.h - CoreFoundation/CFRunLoop.h - CoreFoundation/CFString.h - - CoreFoundation/CFTimeZone.h + - CoreFoundation/CFTimeZone.h - CoreFoundation/CFUUID.h - CoreFoundation/CoreFoundation.h - CoreGraphics/CoreGraphics.h diff --git a/build/sysincl/intrinsic.yml b/build/sysincl/intrinsic.yml index 2e89f67c48..9355b61c13 100644 --- a/build/sysincl/intrinsic.yml +++ b/build/sysincl/intrinsic.yml @@ -32,11 +32,11 @@ - rtmintrin.h - s390intrin.h - sanitizer/asan_interface.h - - sanitizer/common_interface_defs.h + - sanitizer/common_interface_defs.h - sanitizer/coverage_interface.h - sanitizer/lsan_interface.h - sanitizer/msan_interface.h - - sanitizer/tsan_interface.h + - sanitizer/tsan_interface.h - sanitizer/tsan_interface_atomic.h - smmintrin.h - tbmintrin.h diff --git a/build/sysincl/misc.yml b/build/sysincl/misc.yml index 28adc3a0a6..e9e6095888 100644 --- a/build/sysincl/misc.yml +++ b/build/sysincl/misc.yml @@ -420,20 +420,20 @@ - source_filter: "^contrib/libs/dpdk/" includes: - process.h: contrib/libs/dpdk/drivers/bus/dpaa/include/process.h - + - source_filter: "^contrib/libs/dpdk_arm64/" includes: - process.h: contrib/libs/dpdk_arm64/drivers/bus/dpaa/include/process.h -- source_filter: "^contrib/libs/usrsctp/" - includes: - - netinet/sctp_uio.h: contrib/libs/usrsctp/usrsctplib/netinet/sctp_uio.h - -- source_filter: "^contrib/libs/libsrtp/" - includes: - - alloc.h: contrib/libs/libsrtp/crypto/include/alloc.h - - err.h: contrib/libs/libsrtp/crypto/include/err.h - - stat.h: contrib/libs/libsrtp/crypto/include/stat.h +- source_filter: "^contrib/libs/usrsctp/" + includes: + - netinet/sctp_uio.h: contrib/libs/usrsctp/usrsctplib/netinet/sctp_uio.h + +- source_filter: "^contrib/libs/libsrtp/" + includes: + - alloc.h: contrib/libs/libsrtp/crypto/include/alloc.h + - err.h: contrib/libs/libsrtp/crypto/include/err.h + - stat.h: contrib/libs/libsrtp/crypto/include/stat.h - source_filter: "^contrib/restricted/openal-soft" includes: diff --git a/build/sysincl/stl-to-libcxx.yml b/build/sysincl/stl-to-libcxx.yml index 66b3dac844..7d9600f345 100644 --- a/build/sysincl/stl-to-libcxx.yml +++ b/build/sysincl/stl-to-libcxx.yml @@ -48,7 +48,7 @@ - __config: contrib/libs/cxxsupp/libcxx/include/__config - __undef_macros: contrib/libs/cxxsupp/libcxx/include/__undef_macros - algorithm: contrib/libs/cxxsupp/libcxx/include/algorithm - - any: contrib/libs/cxxsupp/libcxx/include/any + - any: contrib/libs/cxxsupp/libcxx/include/any - array: contrib/libs/cxxsupp/libcxx/include/array - atomic: contrib/libs/cxxsupp/libcxx/include/atomic - bit: contrib/libs/cxxsupp/libcxx/include/bit diff --git a/build/sysincl/stl-to-nothing.yml b/build/sysincl/stl-to-nothing.yml index 62c534b1f4..c7e27d1411 100644 --- a/build/sysincl/stl-to-nothing.yml +++ b/build/sysincl/stl-to-nothing.yml @@ -31,7 +31,7 @@ - __config - __undef_macros - algorithm - - any + - any - array - atomic - bit diff --git a/build/sysincl/unsorted.yml b/build/sysincl/unsorted.yml index bd33e5853c..a889b923a0 100644 --- a/build/sysincl/unsorted.yml +++ b/build/sysincl/unsorted.yml @@ -266,8 +266,8 @@ - unlocked-io.h - cygwin/version.h - zircon/syscalls.h - - zircon/types.h - - zircon/process.h + - zircon/types.h + - zircon/process.h - sys/linux-syscalls.h - tr1/memory - android/log.h @@ -289,21 +289,21 @@ - machine/fpu.h - malloc_np.h - netinet/sctp_uio.h - - netinet/ip_options.h - - netinet/sctp_kdtrace.h - - netinet/sctp_lock_bsd.h - - netinet/sctp_lock_empty.h - - netinet/sctp_os_bsd.h - - netinet6/in6_pcb.h - - netinet6/ip6_var.h - - netinet6/ip6protosw.h - - netinet6/scope6_var.h - - netipsec/ipsec.h - - netipsec/ipsec6.h - - netipsec/key.h + - netinet/ip_options.h + - netinet/sctp_kdtrace.h + - netinet/sctp_lock_bsd.h + - netinet/sctp_lock_empty.h + - netinet/sctp_os_bsd.h + - netinet6/in6_pcb.h + - netinet6/ip6_var.h + - netinet6/ip6protosw.h + - netinet6/scope6_var.h + - netipsec/ipsec.h + - netipsec/ipsec6.h + - netipsec/key.h - opie.h - osreldate.h - - opt_sctp.h + - opt_sctp.h - pthread_np.h - ripemd.h - sha.h @@ -313,22 +313,22 @@ - sys/cpuset.h - sys/disklabel.h - sys/endian.h - - sys/eventhandler.h + - sys/eventhandler.h - sys/exec.h - sys/extattr.h - - sys/filedesc.h - - sys/gsb_crc32.h + - sys/filedesc.h + - sys/gsb_crc32.h - sys/jail.h - sys/ktrace.h - sys/machine.h - sys/module.h - sys/mpctl.h - sys/mutex.h - - sys/rwlock.h + - sys/rwlock.h - sys/semaphore.h - - sys/smp.h + - sys/smp.h - sys/stdint.h - - sys/systm.h + - sys/systm.h - sys/thr.h - sys/timespec.h - sys/umtx.h @@ -613,7 +613,7 @@ - tpf/sysapi.h #endif - asm/page.h - - asm/sgidefs.h + - asm/sgidefs.h - wrl\client.h - wrl\event.h - wrl\wrappers\corewrappers.h diff --git a/build/sysincl/windows.yml b/build/sysincl/windows.yml index a6858cb70f..26ae7b5bfa 100644 --- a/build/sysincl/windows.yml +++ b/build/sysincl/windows.yml @@ -164,8 +164,8 @@ - oleacc.h - oleauto.h - olectl.h - - packon.h - - packoff.h + - packon.h + - packoff.h - pathcch.h - p2p.h - pdh.h @@ -305,7 +305,7 @@ - ws2bth.h - ws2def.h - ws2ipdef.h - - ws2spi.h + - ws2spi.h - ws2tcpip.h - wscapi.h - wspiapi.h diff --git a/build/ya.conf.json b/build/ya.conf.json index a93447e052..5f7cc875d6 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -1074,14 +1074,14 @@ } ] }, - "maven_import_sandbox_uploader": { - "tools": { + "maven_import_sandbox_uploader": { + "tools": { "maven_import_sandbox_uploader": { "bottle": "maven_import_sandbox_uploader", "executable": "maven_import_sandbox_uploader" } - }, - "platforms": [ + }, + "platforms": [ { "host": { "os": "LINUX" @@ -1107,8 +1107,8 @@ }, "default": true } - ] - }, + ] + }, "arc": { "tools": { "arc": { @@ -5784,17 +5784,17 @@ ] } }, - "maven_import_sandbox_uploader": { - "formula": { + "maven_import_sandbox_uploader": { + "formula": { "sandbox_id": 1031028343, - "match": "uploader" - }, - "executable": { + "match": "uploader" + }, + "executable": { "maven_import_sandbox_uploader": [ "uploader" ] - } - }, + } + }, "arc": { "formula": { "sandbox_id": [ diff --git a/build/ymake.core.conf b/build/ymake.core.conf index de815934b4..081833998b 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -936,7 +936,7 @@ otherwise { macro _JAVA_PROTO_CMD(File) { .CMD=${cwd;rootdir;input:File} $YMAKE_PYTHON ${input:"build/scripts/tared_protoc.py"} --tar-output ${output;norel;nopath;noext;suf=.jsrc:File} --protoc-out-dir $ARCADIA_BUILD_ROOT/java_out $PROTOC -I=./$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_ROOT $_JAVA_PROTO_OUT_ARG$ARCADIA_BUILD_ROOT/java_out ${input;rootrel:File} ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} $JAVA_PROTO_ARGS } - + # tag:proto tag:perl-specific tag:deprecated ### @usage: XS_PROTO(InputProto Dir Outputs...) # deprecated @@ -966,8 +966,8 @@ when ($PY_PROTOS_FOR == "yes") { macro _PY_EVLOG_CMD_BASE(File, Suf, Args...) { .CMD=$_PY_PROTO_CMD_BASE($File $Suf $Args) .PEERDIR=library/cpp/eventlog/proto -} - +} + # tag:python-specific tag:proto macro _PY_EVLOG_CMD(File) { .CMD=${cwd;rootdir;input:File} $_PY_EVLOG_CMD_BASE($File _ev_pb2.py) @@ -982,8 +982,8 @@ macro _PY_EVLOG_CMD_INTERNAL(File) { macro _JAVA_EVLOG_CMD(File) { .CMD=$COPY_CMD ${input:File} ${output;nopath;noext;norel;suf=_ev.proto:File} ${kv;hide:"p EV"} ${kv;hide:"pc yellow"} .PEERDIR=library/cpp/eventlog/proto -} - +} + # tag:sanitize RUN_NO_SANITIZE=$YMAKE_PYTHON ${input:"build/scripts/run_tool.py"} -- @@ -2461,8 +2461,8 @@ module LIBRARY: _LIBRARY { SET(MODULE_LANG CPP) ADD_CLANG_TIDY() -} - +} + # tag:internal ### @usage: _BARE_MODULE() # internal ### @@ -2980,8 +2980,8 @@ module _DLL_COMPATIBLE_LIBRARY: LIBRARY { # as SRCS in packages use macro BUNDLE_SRCS! -PACKED_PACKAGE_ARGS= -PACKED_PACKAGE_EXT= +PACKED_PACKAGE_ARGS= +PACKED_PACKAGE_EXT= ### @usage: PACK(archive_type) ### ### When placed inside the PACKAGE module, packs the build results tree to the archive with specified extension. Currently supported extensions are `tar` and `tar.gz` @@ -2991,8 +2991,8 @@ PACKED_PACKAGE_EXT= ### @see: [PACKAGE()](#module_PACKAGE) macro PACK(Ext) { SET(PACKED_PACKAGE_EXT $Ext) -} - +} + PACKAGE_STRICT_VALUE= macro PACKAGE_STRICT() { SET(PACKAGE_STRICT_VALUE yes) @@ -3416,7 +3416,7 @@ module EXTERNAL_JAVA_LIBRARY: _BASE_UNIT { .ALIASES=SRCS=_SRCS_NO_GLOBAL .ALLOWED=EMBED_JAVA_VCS_INFO .RESTRICTED=EXTERNAL_JAR - + PEERDIR(build/platform/java/jdk) PEERDIR+=$JDK_RESOURCE_PEERDIR @@ -3426,17 +3426,17 @@ module EXTERNAL_JAVA_LIBRARY: _BASE_UNIT { MACRO_ALIAS(PROTO_CMD _JAVA_PROTO_CMD) MACRO_ALIAS(EVLOG_CMD _JAVA_EVLOG_CMD) MACRO_ALIAS(FBS_CMD _JAVA_FLATC_CMD) - + when ($PACKAGE_PREFIX) { PACKAGE_PREFIX_ARGS=--package-prefix=$PACKAGE_PREFIX } - + DISABLE(NEED_PLATFORM_PEERDIRS) NO_PLATFORM() SET(MODULE_SUFFIX .jar) SET(MODULE_LANG JAVA) -} - +} + # tag:java-specific ALL_JAR_SOURCES= LINT_JAVA_SOURCES= @@ -4050,8 +4050,8 @@ module PY2_LIBRARY: _LIBRARY { PY_PROTO_MYPY_PLUGIN_INTERNAL=$PY_PROTO_MYPY_PLUGIN_BASE ${output;hide;noauto;norel;nopath;noext;suf=__int___pb2.pyi:File} ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__int___pb2.pyi:File} ${nopath;noext;suf=_pb2.pyi:File}"}) } SET(MODULE_LANG PY2) -} - +} + ### @usage: _PY3_LIBRARY() ### @@ -4148,8 +4148,8 @@ module _BASE_PY_PROGRAM: _BASE_PROGRAM { .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER USE_PYTHON2 .RESTRICTED=PYTHON3_ADDINCL USE_PYTHON3 PYTHON2_ADDINCL OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS SET(MODULE_LANG PY2) -} - +} + # tag:python-specific tag:codenav when ($CODENAVIGATION && $NOCODENAVIGATION != "yes") { PY3_PROGRAM_LINK_EXE=$LINK_EXE ${kv;hide:"py3yndex $TARGET"} @@ -4420,7 +4420,7 @@ when ($JDK_VERSION == "8") { # tag:java-specific COMPILE_JAVA=${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON ${input:"build/scripts/compile_java.py"} --javac-bin $JDK_RESOURCE/bin/javac --jar-bin $JDK_RESOURCE/bin/jar $JAVA_VCS_MF_ARG $PACKAGE_PREFIX_ARGS --jar-output $TARGET --srcs-jar-output ${output;suf=-sources.jar:REALPRJNAME} $AUTO_INPUT DELIM $JAVAC_OPTS DELIM $PEERS ${requirements;hide:JAVA_REQUIREMENTS} ${kv;hide:"p JV"} ${kv;hide:"pc light-blue"} ${kv;hide:"show_out"} ${kv;hide:"PEERS $PEERS"} REAL_SWIG_DLL_JAR_CMD=$GENERATE_VCS_JAVA_INFO_NODEP && ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON ${input:"build/scripts/build_dll_and_java.py"} $JAVA_SWIG_DELIM $REAL_LINK_DYN_LIB $JAVA_SWIG_DELIM $COMPILE_JAVA $JAVA_SWIG_DELIM $AUTO_INPUT $JAVA_SWIG_DELIM $TARGET $JAVA_SWIG_DELIM ${output;suf=.jar:REALPRJNAME} $JAVA_SWIG_DELIM ${output;suf=-sources.jar:REALPRJNAME} $JAVA_SWIG_DELIM $ARCADIA_BUILD_ROOT $ARCADIA_ROOT - + # tag:deprecated ### @usage: METAQUERY() #deprecated ### @@ -4433,9 +4433,9 @@ module METAQUERY: _BASE_UNIT { PRINT_MODULE_TYPE(METAQUERY $MODULE_PREFIX$REALPRJNAME) } -ARGS_DELIM="MACRO_CALLS_DELIM" - -SYSTEM_PROPERTIES_VALUE= +ARGS_DELIM="MACRO_CALLS_DELIM" + +SYSTEM_PROPERTIES_VALUE= ### @usage: SYSTEM_PROPERTIES([<Key Value>...] [<File Path>...]) ### ### List of Key,Value pairs that will be available to test via System.getProperty(). @@ -4444,10 +4444,10 @@ SYSTEM_PROPERTIES_VALUE= ### Documentation: https://wiki.yandex-team.ru/yatool/test/ macro SYSTEM_PROPERTIES(Args...) { SET_APPEND(SYSTEM_PROPERTIES_VALUE $Args) -} - +} + # tag:java-specific -JVM_ARGS_VALUE= +JVM_ARGS_VALUE= ### @usage: JVM_ARGS(Args...) ### ### Arguments to run Java programs in tests. @@ -4455,8 +4455,8 @@ JVM_ARGS_VALUE= ### Documentation: https://wiki.yandex-team.ru/yatool/test/ macro JVM_ARGS(Args...) { SET_APPEND(JVM_ARGS_VALUE $Args) -} - +} + # tag:java-specific CHECK_JAVA_DEPS_VALUE= ### @usage: CHECK_JAVA_DEPS(<yes|no>) @@ -4476,7 +4476,7 @@ macro USE_ERROR_PRONE() { SET(ERROR_PRONE_VALUE yes) } -TEST_CWD_VALUE= +TEST_CWD_VALUE= ### @usage: TEST_CWD(path) ### ### Defines working directory for test runs. Othen used in conjunction with DATA() macro. @@ -4485,8 +4485,8 @@ TEST_CWD_VALUE= ### Documentation: https://wiki.yandex-team.ru/yatool/test/ macro TEST_CWD(Arg) { SET(TEST_CWD_VALUE $Arg) -} - +} + # tag:java-specific MAKE_UBERJAR_VALUE= ### @usage: UBERJAR() @@ -4640,15 +4640,15 @@ macro IDEA_MODULE_NAME(Name) { } # tag:java-specific -JAVAC_FLAGS_VALUE= +JAVAC_FLAGS_VALUE= ### @usage: JAVAC_FLAGS(Args...) ### ### Set additional Java compilation flags. macro JAVAC_FLAGS(Args...) { SET_APPEND(JAVAC_OPTS $Args) SET_APPEND(JAVAC_FLAGS_VALUE $ARGS_DELIM $Args) -} - +} + # tag:java-specific ENABLE_PREVIEW_VALUE= ### @usage: ENABLE_PREVIEW() @@ -4682,26 +4682,26 @@ macro MAVEN_GROUP_ID(Arg) { } # tag:java-specific -ANNOTATION_PROCESSOR_VALUE= +ANNOTATION_PROCESSOR_VALUE= ### @usage: ANNOTATION_PROCESSOR(processors...) ### ### The macro is in development. ### Used to specify annotation processors to build JAVA_PROGRAM() and JAVA_LIBRARY(). macro ANNOTATION_PROCESSOR(Args...) { SET_APPEND(ANNOTATION_PROCESSOR_VALUE $ARGS_DELIM $Args) -} - -EXCLUDE_VALUE= +} + +EXCLUDE_VALUE= ### EXCLUDE(prefixes) ### ### The macro is in development. ### Specifies which libraries should be excluded from the classpath. macro EXCLUDE(Args...) { SET_APPEND(EXCLUDE_VALUE $ARGS_DELIM $Args) -} - +} + # tag:java-specific -JAVA_SRCS_VALUE= +JAVA_SRCS_VALUE= ### @usage: JAVA_SRCS(srcs) ### ### Specify java source files and resources. A macro can be contained in any of four java modules. @@ -4728,8 +4728,8 @@ JAVA_SRCS_VALUE= ### Documentation: https://wiki.yandex-team.ru/yatool/java/#javasrcs macro JAVA_SRCS(Args...) { SET_APPEND(JAVA_SRCS_VALUE $ARGS_DELIM $Args) -} - +} + # tag:java-specific JAVA_ADD_DLLS_VALUE=no macro ADD_DLLS_TO_JAR() { @@ -4752,7 +4752,7 @@ TEST_CLASSPATH_VALUE= TEST_CLASSPATH_MANAGED= # tag:java-specific -DEPENDENCY_MANAGEMENT_VALUE= +DEPENDENCY_MANAGEMENT_VALUE= ### @usage: DEPENDENCY_MANAGEMENT(path/to/lib1 path/to/lib2 ...) ### ### Lock version of the library from the contrib/java at some point, so that all unversioned PEERDIRs to this library refer to the specified version. @@ -4773,8 +4773,8 @@ DEPENDENCY_MANAGEMENT_VALUE= macro DEPENDENCY_MANAGEMENT(Args...) { SET_APPEND(DEPENDENCY_MANAGEMENT_VALUE $ARGS_DELIM $Args) _GHOST_PEERDIR($Args) -} - +} + # tag:java-specific WITH_JDK_VALUE= ### @usage: WITH_JDK() @@ -4921,8 +4921,8 @@ module _JAVA_PLACEHOLDER: _BASE_UNIT { DEPENDENCY_MANAGEMENT(contrib/java/com/google/protobuf/protobuf-java-util/${JAVA_PROTO_RUNTIME_VERSION}) CHECK_DEPENDENT_DIRS(DENY PEERDIRS contrib/java/com/google/protobuf/protobuf-java-util EXCEPT contrib/java/com/google/protobuf/protobuf-java-util/${JAVA_PROTO_RUNTIME_VERSION}) SET(MODULE_LANG JAVA) -} - +} + # tag:java-specific JAVA_CLASSPATH_CMD_TYPE_VALUE= @@ -7113,7 +7113,7 @@ when ($MSVC == "yes") { otherwise { C_AS_CXX_FLAGS=-x c++ -std=c++17 } - + # tag:cpp-specific ### @usage: COMPILE_C_AS_CXX() ### diff --git a/build/ymake_conf.py b/build/ymake_conf.py index b5f9ae5655..30219eb85e 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -55,7 +55,7 @@ class Platform(object): self.is_intel = self.is_x86 or self.is_x86_64 self.is_armv7 = self.arch in ('armv7', 'armv7a', 'armv7a_neon', 'arm', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53') - self.is_armv8 = self.arch in ('armv8', 'armv8a', 'arm64', 'aarch64', 'armv8a_cortex_a35', 'armv8a_cortex_a53') + self.is_armv8 = self.arch in ('armv8', 'armv8a', 'arm64', 'aarch64', 'armv8a_cortex_a35', 'armv8a_cortex_a53') self.is_armv8m = self.arch in ('armv8m_cortex_m33',) self.is_arm64 = self.arch in ('arm64',) self.is_arm = self.is_armv7 or self.is_armv8 or self.is_armv8m @@ -70,10 +70,10 @@ class Platform(object): self.armv7_float_abi = 'softfp' self.is_cortex_a9 = self.arch in ('armv7a_cortex_a9',) - self.is_cortex_a35 = self.arch in ('armv7ahf_cortex_a35', 'armv8a_cortex_a35') - self.is_cortex_a53 = self.arch in ('armv7ahf_cortex_a53', 'armv8a_cortex_a53') + self.is_cortex_a35 = self.arch in ('armv7ahf_cortex_a35', 'armv8a_cortex_a35') + self.is_cortex_a53 = self.arch in ('armv7ahf_cortex_a53', 'armv8a_cortex_a53') self.is_cortex_m33 = self.arch in ('armv8m_cortex_m33',) - + self.is_power8le = self.arch == 'ppc64le' self.is_power9le = self.arch == 'power9le' self.is_powerpc = self.is_power8le or self.is_power9le @@ -150,7 +150,7 @@ class Platform(object): (self.is_i686, 'ARCH_I686'), (self.is_x86_64, 'ARCH_X86_64'), (self.is_armv7, 'ARCH_ARM7'), - (self.is_armv7_neon, 'ARCH_ARM7_NEON'), + (self.is_armv7_neon, 'ARCH_ARM7_NEON'), (self.is_armv8, 'ARCH_ARM64'), (self.is_armv8m, 'ARCH_ARM8M'), (self.is_arm, 'ARCH_ARM'), @@ -783,7 +783,7 @@ when (($USEMPROF == "yes") || ($USE_MPROF == "yes")) { for variable in self.platform.os_variables: emit(variable, 'yes') - + if self.platform.is_armv7: emit('ARM7_FLOAT_ABI', self.platform.armv7_float_abi) @@ -796,7 +796,7 @@ when (($USEMPROF == "yes") || ($USE_MPROF == "yes")) { self.print_linux_const() elif self.platform.is_windows: self.print_windows_target_const() - + def print_host_settings(self): emit('HOST_PLATFORM', self.platform.os_compat) if not self.platform.is_windows: @@ -1168,13 +1168,13 @@ class GnuToolchain(Toolchain): (target.is_apple and target.is_x86_64, 'x86_64-apple-darwin14'), (target.is_apple and target.is_macos_arm64, 'arm64-apple-macos11'), (target.is_apple and target.is_armv7, 'armv7-apple-darwin14'), - (target.is_apple and target.is_armv8, 'arm64-apple-darwin14'), + (target.is_apple and target.is_armv8, 'arm64-apple-darwin14'), (target.is_yocto and target.is_armv7, 'arm-poky-linux-gnueabi'), (target.is_android and target.is_x86, 'i686-linux-android'), (target.is_android and target.is_x86_64, 'x86_64-linux-android'), (target.is_android and target.is_armv7, 'armv7a-linux-androideabi'), (target.is_android and target.is_armv8, 'aarch64-linux-android'), - ]) + ]) if target.is_android: # Android NDK allows specification of API level in target triple, e.g.: @@ -1194,18 +1194,18 @@ class GnuToolchain(Toolchain): if target.is_cortex_a9: self.c_flags_platform.append('-mcpu=cortex-a9') - if target.is_cortex_a35: - self.c_flags_platform.append('-mcpu=cortex-a35') - - elif target.is_cortex_a53: - self.c_flags_platform.append('-mcpu=cortex-a53') - + if target.is_cortex_a35: + self.c_flags_platform.append('-mcpu=cortex-a35') + + elif target.is_cortex_a53: + self.c_flags_platform.append('-mcpu=cortex-a53') + elif target.is_cortex_m33: self.c_flags_platform.append('-mcpu=cortex-m33 -mfpu=fpv5-sp-d16') - elif target.is_armv7_neon: - self.c_flags_platform.append('-mfpu=neon') - + elif target.is_armv7_neon: + self.c_flags_platform.append('-mfpu=neon') + if (target.is_armv7 or target.is_armv8m) and build.is_size_optimized: # Enable ARM Thumb2 variable-length instruction encoding # to reduce code size |