diff options
author | deshevoy <deshevoy@yandex-team.ru> | 2022-02-10 16:46:57 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:57 +0300 |
commit | 28148f76dbfcc644d96427d41c92f36cbf2fdc6e (patch) | |
tree | b83306b6e37edeea782e9eed673d89286c4fef35 /build | |
parent | e988f30484abe5fdeedcc7a5d3c226c01a21800c (diff) | |
download | ydb-28148f76dbfcc644d96427d41c92f36cbf2fdc6e.tar.gz |
Restoring authorship annotation for <deshevoy@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'build')
-rw-r--r-- | build/platform/msvc/ya.make | 2 | ||||
-rw-r--r-- | build/plugins/linker_script.py | 24 | ||||
-rw-r--r-- | build/plugins/pybuild.py | 122 | ||||
-rw-r--r-- | build/plugins/ytest.py | 8 | ||||
-rwxr-xr-x | build/scripts/fetch_from.py | 442 | ||||
-rw-r--r-- | build/scripts/fetch_from_mds.py | 72 | ||||
-rwxr-xr-x | build/scripts/fetch_from_sandbox.py | 20 | ||||
-rw-r--r-- | build/scripts/gen_py3_reg.py | 62 | ||||
-rw-r--r-- | build/scripts/gen_py_protos.py | 56 | ||||
-rw-r--r-- | build/scripts/gen_tasklet_reg.py | 54 | ||||
-rwxr-xr-x | build/scripts/py_compile.py | 2 | ||||
-rw-r--r-- | build/scripts/ya.make | 2 | ||||
-rw-r--r-- | build/sysincl/misc-win.yml | 4 | ||||
-rw-r--r-- | build/ya.conf.json | 228 | ||||
-rw-r--r-- | build/ymake.core.conf | 214 | ||||
-rwxr-xr-x | build/ymake_conf.py | 76 |
16 files changed, 694 insertions, 694 deletions
diff --git a/build/platform/msvc/ya.make b/build/platform/msvc/ya.make index 5ceb18cb23..ccb602c280 100644 --- a/build/platform/msvc/ya.make +++ b/build/platform/msvc/ya.make @@ -13,7 +13,7 @@ ELSE() ENDIF() IF (CLANG_CL) - DECLARE_EXTERNAL_RESOURCE(MSVC_FOR_CLANG sbr:1383387533) # Microsoft Visual C++ 2017 14.16.27023 (15.9.5) + DECLARE_EXTERNAL_RESOURCE(MSVC_FOR_CLANG sbr:1383387533) # Microsoft Visual C++ 2017 14.16.27023 (15.9.5) ENDIF() END() diff --git a/build/plugins/linker_script.py b/build/plugins/linker_script.py index 81e263ed5f..bee9777a4e 100644 --- a/build/plugins/linker_script.py +++ b/build/plugins/linker_script.py @@ -1,12 +1,12 @@ -def onlinker_script(unit, *args): - """ - @usage: LINKER_SCRIPT(Files...) - - Specify files to be used as a linker script - """ - for arg in args: - if not arg.endswith(".ld") and not arg.endswith(".ld.in"): - unit.message(['error', "Invalid linker script extension: {}".format(arg)]) - return - - unit.onglobal_srcs(list(args)) +def onlinker_script(unit, *args): + """ + @usage: LINKER_SCRIPT(Files...) + + Specify files to be used as a linker script + """ + for arg in args: + if not arg.endswith(".ld") and not arg.endswith(".ld.in"): + unit.message(['error', "Invalid linker script extension: {}".format(arg)]) + return + + unit.onglobal_srcs(list(args)) diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index 8484a72633..f32a2d39a0 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -35,19 +35,19 @@ def uniq_suffix(path, unit): return '' return '.{}'.format(pathid(path)[:4]) -def pb2_arg(suf, path, mod, unit): - return '{path}__int__{suf}={mod}{modsuf}'.format( - path=stripext(to_build_root(path, unit)), - suf=suf, - mod=mod, - modsuf=stripext(suf) - ) +def pb2_arg(suf, path, mod, unit): + return '{path}__int__{suf}={mod}{modsuf}'.format( + path=stripext(to_build_root(path, unit)), + suf=suf, + mod=mod, + modsuf=stripext(suf) + ) def proto_arg(path, mod, unit): return '{}.proto={}'.format(stripext(to_build_root(path, unit)), mod) -def pb_cc_arg(suf, path, unit): - return '{}{suf}'.format(stripext(to_build_root(path, unit)), suf=suf) +def pb_cc_arg(suf, path, unit): + return '{}{suf}'.format(stripext(to_build_root(path, unit)), suf=suf) def ev_cc_arg(path, unit): return '{}.ev.pb.cc'.format(stripext(to_build_root(path, unit))) @@ -143,23 +143,23 @@ def add_python_lint_checks(unit, py_ver, files): unit.onadd_check(["flake8.py{}".format(py_ver), flake8_cfg] + resolved_files) -def is_py3(unit): - return unit.get("PYTHON3") == "yes" - - +def is_py3(unit): + return unit.get("PYTHON3") == "yes" + + def on_py_program(unit, *args): py_program(unit, is_py3(unit)) -def py_program(unit, py3): +def py_program(unit, py3): """ Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/#modulpyprogramimakrospymain """ - if py3: + if py3: peers = ['library/python/runtime_py3/main'] if unit.get('PYTHON_SQLITE3') != 'no': peers.append('contrib/tools/python3/src/Modules/_sqlite') - else: + else: peers = ['library/python/runtime/main'] if unit.get('PYTHON_SQLITE3') != 'no': peers.append('contrib/tools/python/src/Modules/_sqlite') @@ -193,7 +193,7 @@ def onpy_srcs(unit, *args): # and "modname" will be used as a module name. upath = unit.path()[3:] - py3 = is_py3(unit) + py3 = is_py3(unit) py_main_only = unit.get('PROCESS_PY_MAIN_ONLY') with_py = not unit.get('PYBUILD_NO_PY') with_pyc = not unit.get('PYBUILD_NO_PYC') @@ -207,7 +207,7 @@ def onpy_srcs(unit, *args): unit_needs_main = unit.get('MODULE_TYPE') in ('PROGRAM', 'DLL') if unit_needs_main: - py_program(unit, py3) + py_program(unit, py3) py_namespace_value = unit.get('PY_NAMESPACE_VALUE') if py_namespace_value == ".": @@ -285,7 +285,7 @@ def onpy_srcs(unit, *args): arg = next(args) if '=' in arg: - main_py = False + main_py = False path, mod = arg.split('=', 1) else: if trim: @@ -295,7 +295,7 @@ def onpy_srcs(unit, *args): path = '{}.proto'.format(arg[:-9]) else: path = arg - main_py = (path == '__main__.py' or path.endswith('/__main__.py')) + main_py = (path == '__main__.py' or path.endswith('/__main__.py')) if not py3 and unit_needs_main and main_py: mod = '__main__' else: @@ -313,9 +313,9 @@ def onpy_srcs(unit, *args): mod = ns + mod_name if main_mod: - py_main(unit, mod + ":main") + py_main(unit, mod + ":main") elif py3 and unit_needs_main and main_py: - py_main(unit, mod) + py_main(unit, mod) if py_main_only: continue @@ -406,7 +406,7 @@ def onpy_srcs(unit, *args): ] + cython_directives cython(cython_args) - py_register(unit, mod, py3) + py_register(unit, mod, py3) process_pyx(filename, path, out_suffix, noext) if files2res: @@ -440,9 +440,9 @@ def onpy_srcs(unit, *args): res = [] - if py3: + if py3: mod_list_md5 = md5() - for path, mod in pys: + for path, mod in pys: mod_list_md5.update(mod) if not (venv and is_extended_source_search_enabled(path, unit)): dest = 'py/' + mod.replace('.', '/') + '.py' @@ -463,11 +463,11 @@ def onpy_srcs(unit, *args): ns_res += ['-', '{}="{}"'.format(key, namespaces)] unit.onresource(ns_res) - unit.onresource_files(res) + unit.onresource_files(res) add_python_lint_checks(unit, 3, [path for path, mod in pys] + unit.get(['_PY_EXTRA_LINT_FILES_VALUE']).split()) - else: - for path, mod in pys: - root_rel_path = rootrel_arc_src(path, unit) + else: + for path, mod in pys: + root_rel_path = rootrel_arc_src(path, unit) if with_py: key = '/py_modules/' + mod res += [ @@ -480,9 +480,9 @@ def onpy_srcs(unit, *args): unit.on_py_compile_bytecode([root_rel_path + '-', src, dst]) res += [dst + '.yapyc', '/py_code/' + mod] - unit.onresource(res) + unit.onresource(res) add_python_lint_checks(unit, 2, [path for path, mod in pys] + unit.get(['_PY_EXTRA_LINT_FILES_VALUE']).split()) - + use_vanilla_protoc = unit.get('USE_VANILLA_PROTOC') == 'yes' if use_vanilla_protoc: cpp_runtime_path = 'contrib/libs/protobuf_std' @@ -497,15 +497,15 @@ def onpy_srcs(unit, *args): if not upath.startswith(py_runtime_path) and not upath.startswith(builtin_proto_path): unit.onpeerdir(py_runtime_path) - unit.onpeerdir(unit.get("PY_PROTO_DEPS").split()) + unit.onpeerdir(unit.get("PY_PROTO_DEPS").split()) proto_paths = [path for path, mod in protos] unit.on_generate_py_protos_internal(proto_paths) - unit.onpy_srcs([ - pb2_arg(py_suf, path, mod, unit) - for path, mod in protos - for py_suf in unit.get("PY_PROTO_SUFFIXES").split() - ]) + unit.onpy_srcs([ + pb2_arg(py_suf, path, mod, unit) + for path, mod in protos + for py_suf in unit.get("PY_PROTO_SUFFIXES").split() + ]) if optimize_proto and need_gazetteer_peerdir: unit.onpeerdir(['kernel/gazetteer/proto']) @@ -531,28 +531,28 @@ def _check_test_srcs(*args): def ontest_srcs(unit, *args): _check_test_srcs(*args) - if unit.get('PY3TEST_BIN' if is_py3(unit) else 'PYTEST_BIN') != 'no': + if unit.get('PY3TEST_BIN' if is_py3(unit) else 'PYTEST_BIN') != 'no': unit.onpy_srcs(["NAMESPACE", "__tests__"] + list(args)) -def onpy_doctests(unit, *args): +def onpy_doctests(unit, *args): """ @usage PY_DOCTEST(Packages...) Add to the test doctests for specified Python packages The packages should be part of a test (listed as sources of the test or its PEERDIRs). """ - if unit.get('PY3TEST_BIN' if is_py3(unit) else 'PYTEST_BIN') != 'no': - unit.onresource(['-', 'PY_DOCTEST_PACKAGES="{}"'.format(' '.join(args))]) - - -def py_register(unit, func, py3): - if py3: - unit.on_py3_register([func]) - else: - unit.on_py_register([func]) - - + if unit.get('PY3TEST_BIN' if is_py3(unit) else 'PYTEST_BIN') != 'no': + unit.onresource(['-', 'PY_DOCTEST_PACKAGES="{}"'.format(' '.join(args))]) + + +def py_register(unit, func, py3): + if py3: + unit.on_py3_register([func]) + else: + unit.on_py_register([func]) + + def onpy_register(unit, *args): """ @usage: PY_REGISTER([package.]module_name) @@ -569,28 +569,28 @@ def onpy_register(unit, *args): Documentation: https://wiki.yandex-team.ru/arcadia/python/pysrcs/#makrospyregister """ - py3 = is_py3(unit) + py3 = is_py3(unit) - for name in args: + for name in args: assert '=' not in name, name py_register(unit, name, py3) if '.' in name: shortname = name.rsplit('.', 1)[1] - if py3: + if py3: unit.oncflags(['-DPyInit_{}=PyInit_{}'.format(shortname, mangle(name))]) - else: + else: unit.oncflags(['-Dinit{}=init{}'.format(shortname, mangle(name))]) - - -def py_main(unit, arg): + + +def py_main(unit, arg): if unit.get('IGNORE_PY_MAIN'): return unit_needs_main = unit.get('MODULE_TYPE') in ('PROGRAM', 'DLL') if unit_needs_main: py_program(unit, is_py3(unit)) - unit.onresource(['-', 'PY_MAIN={}'.format(arg)]) - - + unit.onresource(['-', 'PY_MAIN={}'.format(arg)]) + + def onpy_main(unit, arg): """ @usage: PY_MAIN(package.module[:func]) @@ -605,7 +605,7 @@ def onpy_main(unit, arg): if ':' not in arg: arg += ':main' - py_main(unit, arg) + py_main(unit, arg) def onpy_constructor(unit, arg): diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index bad8bcc0b3..8970837f0f 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -38,10 +38,10 @@ PROJECT_TIDY_CONFIG_MAP_PATH = "build/yandex_specific/config/clang_tidy/tidy_pro tidy_config_map = None -def ontest_data(unit, *args): - ymake.report_configure_error("TEST_DATA is removed in favour of DATA") - - +def ontest_data(unit, *args): + ymake.report_configure_error("TEST_DATA is removed in favour of DATA") + + def save_in_file(filepath, data): if filepath: with open(filepath, 'a') as file_handler: diff --git a/build/scripts/fetch_from.py b/build/scripts/fetch_from.py index bff41b3852..db4fea50bf 100755 --- a/build/scripts/fetch_from.py +++ b/build/scripts/fetch_from.py @@ -1,25 +1,25 @@ import datetime as dt import errno -import hashlib +import hashlib import json import logging import os import platform -import random +import random import shutil import socket -import string -import sys +import string +import sys import tarfile import urllib2 - + import retry - - -def make_user_agent(): - return 'fetch_from: {host}'.format(host=socket.gethostname()) - - + + +def make_user_agent(): + return 'fetch_from: {host}'.format(host=socket.gethostname()) + + def add_common_arguments(parser): parser.add_argument('--copy-to') # used by jbuild in fetch_resource parser.add_argument('--rename-to') # used by test_node in inject_mds_resource_to_graph @@ -30,32 +30,32 @@ def add_common_arguments(parser): parser.add_argument('--log-path') parser.add_argument('-v', '--verbose', action='store_true', default=os.environ.get('YA_VERBOSE_FETCHER'), help='increase stderr verbosity') parser.add_argument('outputs', nargs='*', default=[]) - - + + def ensure_dir(path): if not (path == '' or os.path.isdir(path)): os.makedirs(path) # Reference code: library/python/fs/__init__.py -def hardlink_or_copy(src, dst): +def hardlink_or_copy(src, dst): ensure_dir(os.path.dirname(dst)) - if os.name == 'nt': - shutil.copy(src, dst) - else: - try: - os.link(src, dst) - except OSError as e: - if e.errno == errno.EEXIST: - return + if os.name == 'nt': + shutil.copy(src, dst) + else: + try: + os.link(src, dst) + except OSError as e: + if e.errno == errno.EEXIST: + return elif e.errno in (errno.EXDEV, errno.EMLINK, errno.EINVAL, errno.EACCES): sys.stderr.write("Can't make hardlink (errno={}) - fallback to copy: {} -> {}\n".format(e.errno, src, dst)) - shutil.copy(src, dst) - else: - raise - - + shutil.copy(src, dst) + else: + raise + + def rename_or_copy_and_remove(src, dst): ensure_dir(os.path.dirname(dst)) @@ -66,30 +66,30 @@ def rename_or_copy_and_remove(src, dst): os.remove(src) -class BadChecksumFetchError(Exception): - pass - - -class IncompleteFetchError(Exception): - pass - - -class ResourceUnpackingError(Exception): - pass - - -class ResourceIsDirectoryError(Exception): - pass - - -class OutputIsDirectoryError(Exception): - pass - - -class OutputNotExistError(Exception): - pass - - +class BadChecksumFetchError(Exception): + pass + + +class IncompleteFetchError(Exception): + pass + + +class ResourceUnpackingError(Exception): + pass + + +class ResourceIsDirectoryError(Exception): + pass + + +class OutputIsDirectoryError(Exception): + pass + + +class OutputNotExistError(Exception): + pass + + def setup_logging(args, base_name): def makedirs(path): try: @@ -109,11 +109,11 @@ def setup_logging(args, base_name): logging.getLogger().addHandler(logging.StreamHandler(sys.stderr)) -def is_temporary(e): - +def is_temporary(e): + def is_broken(e): return isinstance(e, urllib2.HTTPError) and e.code in (410, 404) - + if is_broken(e): return False @@ -125,98 +125,98 @@ def is_temporary(e): return error.is_temporary_error(e) -def uniq_string_generator(size=6, chars=string.ascii_lowercase + string.digits): - return ''.join(random.choice(chars) for _ in range(size)) - - -def report_to_snowden(value): - def inner(): - body = { - 'namespace': 'ygg', - 'key': 'fetch-from-sandbox', - 'value': json.dumps(value), - } - - urllib2.urlopen( - 'https://back-snowden.qloud.yandex-team.ru/report/add', - json.dumps([body, ]), - timeout=5, - ) - - try: - inner() - except Exception as e: +def uniq_string_generator(size=6, chars=string.ascii_lowercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + + +def report_to_snowden(value): + def inner(): + body = { + 'namespace': 'ygg', + 'key': 'fetch-from-sandbox', + 'value': json.dumps(value), + } + + urllib2.urlopen( + 'https://back-snowden.qloud.yandex-team.ru/report/add', + json.dumps([body, ]), + timeout=5, + ) + + try: + inner() + except Exception as e: logging.warning('report_to_snowden failed: %s', e) - - -def copy_stream(read, *writers, **kwargs): - chunk_size = kwargs.get('size', 1024*1024) - while True: - data = read(chunk_size) - if not data: - break - for write in writers: - write(data) - - -def md5file(fname): - res = hashlib.md5() - with open(fname, 'rb') as f: - copy_stream(f.read, res.update) - return res.hexdigest() - - -def git_like_hash_with_size(filepath): - """ - Calculate git like hash for path - """ - sha = hashlib.sha1() - - file_size = 0 - - with open(filepath, 'rb') as f: - while True: - block = f.read(2 ** 16) - - if not block: - break - - file_size += len(block) - sha.update(block) - - sha.update('\0') - sha.update(str(file_size)) - - return sha.hexdigest(), file_size - - -def size_printer(display_name, size): - sz = [0] - last_stamp = [dt.datetime.now()] - - def printer(chunk): - sz[0] += len(chunk) - now = dt.datetime.now() - if last_stamp[0] + dt.timedelta(seconds=10) < now: - if size: - print >>sys.stderr, "##status##{} - [[imp]]{:.1f}%[[rst]]".format(display_name, 100.0 * sz[0] / size) - last_stamp[0] = now - - return printer - - + + +def copy_stream(read, *writers, **kwargs): + chunk_size = kwargs.get('size', 1024*1024) + while True: + data = read(chunk_size) + if not data: + break + for write in writers: + write(data) + + +def md5file(fname): + res = hashlib.md5() + with open(fname, 'rb') as f: + copy_stream(f.read, res.update) + return res.hexdigest() + + +def git_like_hash_with_size(filepath): + """ + Calculate git like hash for path + """ + sha = hashlib.sha1() + + file_size = 0 + + with open(filepath, 'rb') as f: + while True: + block = f.read(2 ** 16) + + if not block: + break + + file_size += len(block) + sha.update(block) + + sha.update('\0') + sha.update(str(file_size)) + + return sha.hexdigest(), file_size + + +def size_printer(display_name, size): + sz = [0] + last_stamp = [dt.datetime.now()] + + def printer(chunk): + sz[0] += len(chunk) + now = dt.datetime.now() + if last_stamp[0] + dt.timedelta(seconds=10) < now: + if size: + print >>sys.stderr, "##status##{} - [[imp]]{:.1f}%[[rst]]".format(display_name, 100.0 * sz[0] / size) + last_stamp[0] = now + + return printer + + def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1=None, tries=10, writers=None): - logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5) - tmp_file_name = uniq_string_generator() - - request = urllib2.Request(url, headers={'User-Agent': make_user_agent()}) + logging.info('Downloading from url %s name %s and expected md5 %s', url, resource_file_name, expected_md5) + tmp_file_name = uniq_string_generator() + + request = urllib2.Request(url, headers={'User-Agent': make_user_agent()}) req = retry.retry_func(lambda: urllib2.urlopen(request, timeout=30), tries=tries, delay=5, backoff=1.57079) - logging.debug('Headers: %s', req.headers.headers) - expected_file_size = int(req.headers['Content-Length']) - real_md5 = hashlib.md5() - real_sha1 = hashlib.sha1() - - with open(tmp_file_name, 'wb') as fp: + logging.debug('Headers: %s', req.headers.headers) + expected_file_size = int(req.headers['Content-Length']) + real_md5 = hashlib.md5() + real_sha1 = hashlib.sha1() + + with open(tmp_file_name, 'wb') as fp: copy_stream( req.read, fp.write, @@ -225,73 +225,73 @@ def fetch_url(url, unpack, resource_file_name, expected_md5=None, expected_sha1= size_printer(resource_file_name, expected_file_size), *([] if writers is None else writers) ) - - real_md5 = real_md5.hexdigest() - real_file_size = os.path.getsize(tmp_file_name) - real_sha1.update('\0') - real_sha1.update(str(real_file_size)) - real_sha1 = real_sha1.hexdigest() - - if unpack: - tmp_dir = tmp_file_name + '.dir' - os.makedirs(tmp_dir) - with tarfile.open(tmp_file_name, mode="r|gz") as tar: - tar.extractall(tmp_dir) - tmp_file_name = os.path.join(tmp_dir, resource_file_name) - real_md5 = md5file(tmp_file_name) - - logging.info('File size %s (expected %s)', real_file_size, expected_file_size) - logging.info('File md5 %s (expected %s)', real_md5, expected_md5) - logging.info('File sha1 %s (expected %s)', real_sha1, expected_sha1) - - if expected_md5 and real_md5 != expected_md5: - report_to_snowden( - { - 'headers': req.headers.headers, - 'expected_md5': expected_md5, - 'real_md5': real_md5 - } - ) - - raise BadChecksumFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_md5, - expected_md5, - url, - ) - ) - - if expected_sha1 and real_sha1 != expected_sha1: - report_to_snowden( - { - 'headers': req.headers.headers, - 'expected_sha1': expected_sha1, - 'real_sha1': real_sha1 - } - ) - - raise BadChecksumFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_sha1, - expected_sha1, - url, - ) - ) - - if expected_file_size != real_file_size: - report_to_snowden({'headers': req.headers.headers, 'file_size': real_file_size}) - - raise IncompleteFetchError( - 'Downloaded {}, but expected {} for {}'.format( - real_file_size, - expected_file_size, - url, - ) - ) - - return tmp_file_name - - + + real_md5 = real_md5.hexdigest() + real_file_size = os.path.getsize(tmp_file_name) + real_sha1.update('\0') + real_sha1.update(str(real_file_size)) + real_sha1 = real_sha1.hexdigest() + + if unpack: + tmp_dir = tmp_file_name + '.dir' + os.makedirs(tmp_dir) + with tarfile.open(tmp_file_name, mode="r|gz") as tar: + tar.extractall(tmp_dir) + tmp_file_name = os.path.join(tmp_dir, resource_file_name) + real_md5 = md5file(tmp_file_name) + + logging.info('File size %s (expected %s)', real_file_size, expected_file_size) + logging.info('File md5 %s (expected %s)', real_md5, expected_md5) + logging.info('File sha1 %s (expected %s)', real_sha1, expected_sha1) + + if expected_md5 and real_md5 != expected_md5: + report_to_snowden( + { + 'headers': req.headers.headers, + 'expected_md5': expected_md5, + 'real_md5': real_md5 + } + ) + + raise BadChecksumFetchError( + 'Downloaded {}, but expected {} for {}'.format( + real_md5, + expected_md5, + url, + ) + ) + + if expected_sha1 and real_sha1 != expected_sha1: + report_to_snowden( + { + 'headers': req.headers.headers, + 'expected_sha1': expected_sha1, + 'real_sha1': real_sha1 + } + ) + + raise BadChecksumFetchError( + 'Downloaded {}, but expected {} for {}'.format( + real_sha1, + expected_sha1, + url, + ) + ) + + if expected_file_size != real_file_size: + report_to_snowden({'headers': req.headers.headers, 'file_size': real_file_size}) + + raise IncompleteFetchError( + 'Downloaded {}, but expected {} for {}'.format( + real_file_size, + expected_file_size, + url, + ) + ) + + return tmp_file_name + + def chmod(filename, mode): if platform.system().lower() == 'windows': # https://docs.microsoft.com/en-us/windows/win32/fileio/hard-links-and-junctions: @@ -310,13 +310,13 @@ def chmod(filename, mode): def process(fetched_file, file_name, args, remove=True): assert len(args.rename) <= len(args.outputs), ( 'too few outputs to rename', args.rename, 'into', args.outputs) - + # Forbid changes to the loaded resource chmod(fetched_file, 0o444) - if not os.path.isfile(fetched_file): - raise ResourceIsDirectoryError('Resource must be a file, not a directory: %s' % fetched_file) - + if not os.path.isfile(fetched_file): + raise ResourceIsDirectoryError('Resource must be a file, not a directory: %s' % fetched_file) + if args.copy_to: hardlink_or_copy(fetched_file, args.copy_to) if not args.outputs: @@ -333,8 +333,8 @@ def process(fetched_file, file_name, args, remove=True): if args.untar_to: ensure_dir(args.untar_to) # Extract only requested files - try: - with tarfile.open(fetched_file, mode='r:*') as tar: + try: + with tarfile.open(fetched_file, mode='r:*') as tar: inputs = set(map(os.path.normpath, args.rename + args.outputs[len(args.rename):])) members = [entry for entry in tar if os.path.normpath(os.path.join(args.untar_to, entry.name)) in inputs] tar.extractall(args.untar_to, members=members) @@ -342,10 +342,10 @@ def process(fetched_file, file_name, args, remove=True): for root, _, files in os.walk(args.untar_to): for filename in files: chmod(os.path.join(root, filename), 0o444) - except tarfile.ReadError as e: - logging.exception(e) - raise ResourceUnpackingError('File {} cannot be untared'.format(fetched_file)) - + except tarfile.ReadError as e: + logging.exception(e) + raise ResourceUnpackingError('File {} cannot be untared'.format(fetched_file)) + for src, dst in zip(args.rename, args.outputs): if src == 'RESOURCE': src = fetched_file @@ -360,7 +360,7 @@ def process(fetched_file, file_name, args, remove=True): rename_or_copy_and_remove(src, dst) else: hardlink_or_copy(src, dst) - + for path in args.outputs: if not os.path.exists(path): raise OutputNotExistError('Output does not exist: %s' % os.path.abspath(path)) diff --git a/build/scripts/fetch_from_mds.py b/build/scripts/fetch_from_mds.py index a498662523..5e4e656394 100644 --- a/build/scripts/fetch_from_mds.py +++ b/build/scripts/fetch_from_mds.py @@ -1,50 +1,50 @@ -import os -import sys -import logging +import os +import sys +import logging import argparse - -import fetch_from - -MDS_PREFIX = "https://storage.yandex-team.ru/get-devtools/" - - -def parse_args(): + +import fetch_from + +MDS_PREFIX = "https://storage.yandex-team.ru/get-devtools/" + + +def parse_args(): parser = argparse.ArgumentParser() fetch_from.add_common_arguments(parser) - + parser.add_argument('--key', required=True) - - return parser.parse_args() - - -def fetch(key): - parts = key.split("/") - if len(parts) != 3: - raise ValueError("Invalid MDS key '{}'".format(key)) - - _, sha1, file_name = parts - - fetched_file = fetch_from.fetch_url(MDS_PREFIX + key, False, file_name, expected_sha1=sha1) - - return fetched_file, file_name - - + + return parser.parse_args() + + +def fetch(key): + parts = key.split("/") + if len(parts) != 3: + raise ValueError("Invalid MDS key '{}'".format(key)) + + _, sha1, file_name = parts + + fetched_file = fetch_from.fetch_url(MDS_PREFIX + key, False, file_name, expected_sha1=sha1) + + return fetched_file, file_name + + def main(args): fetched_file, resource_file_name = fetch(args.key) - + fetch_from.process(fetched_file, resource_file_name, args) - - -if __name__ == '__main__': + + +if __name__ == '__main__': args = parse_args() fetch_from.setup_logging(args, os.path.basename(__file__)) - - try: + + try: main(args) - except Exception as e: - logging.exception(e) + except Exception as e: + logging.exception(e) print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() + sys.stderr.flush() import error sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py index fdc3851693..a99542e174 100755 --- a/build/scripts/fetch_from_sandbox.py +++ b/build/scripts/fetch_from_sandbox.py @@ -10,9 +10,9 @@ import time import urllib2 import uuid -import fetch_from +import fetch_from + - ORIGIN_SUFFIX = '?origin=fetch-from-sandbox' MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' TEMPORARY_ERROR_CODES = (429, 500, 503, 504) @@ -68,7 +68,7 @@ def download_by_skynet(resource_info, file_name): if not skynet_id: raise ValueError("Resource does not have skynet_id") - temp_dir = os.path.abspath(fetch_from.uniq_string_generator()) + temp_dir = os.path.abspath(fetch_from.uniq_string_generator()) os.mkdir(temp_dir) sky_get(skynet_id, temp_dir) return os.path.join(temp_dir, file_name) @@ -132,7 +132,7 @@ def fetch_via_script(script, resource_id): return subprocess.check_output([script, str(resource_id)]).rstrip() -def fetch(resource_id, custom_fetcher): +def fetch(resource_id, custom_fetcher): try: resource_info = get_resource_info(resource_id, touch=True, no_links=True) except Exception as e: @@ -179,9 +179,9 @@ def fetch(resource_id, custom_fetcher): if mds_link is not None: # Don't try too hard here: we will get back to MDS later on yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5, tries=2) - yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5) + yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5) if mds_link is not None: - yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5) + yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5) if resource_info.get('attributes', {}).get('ttl') != 'inf': sys.stderr.write('WARNING: resource {} ttl is not "inf".\n'.format(resource_id)) @@ -211,7 +211,7 @@ def fetch(resource_id, custom_fetcher): else: raise Exception("No available protocol and/or server to fetch resource") - return fetched_file, resource_info['file_name'] + return fetched_file, resource_info['file_name'] def _get_resource_info_from_file(resource_file): @@ -241,7 +241,7 @@ def _get_resource_info_from_file(resource_file): def main(args): - custom_fetcher = os.environ.get('YA_CUSTOM_FETCHER') + custom_fetcher = os.environ.get('YA_CUSTOM_FETCHER') resource_info = _get_resource_info_from_file(args.resource_file) if resource_info: @@ -252,8 +252,8 @@ def main(args): fetched_file, file_name = fetch(args.resource_id, custom_fetcher) fetch_from.process(fetched_file, file_name, args, remove=not custom_fetcher and not resource_info) - - + + if __name__ == '__main__': args = parse_args() fetch_from.setup_logging(args, os.path.basename(__file__)) diff --git a/build/scripts/gen_py3_reg.py b/build/scripts/gen_py3_reg.py index 151679b88b..149c094898 100644 --- a/build/scripts/gen_py3_reg.py +++ b/build/scripts/gen_py3_reg.py @@ -1,34 +1,34 @@ -import sys - -template = ''' +import sys + +template = ''' struct PyObject; extern "C" int PyImport_AppendInittab(const char* name, PyObject* (*initfunc)()); extern "C" PyObject* {1}(); - -namespace { - struct TRegistrar { - inline TRegistrar() { - // TODO Collect all modules and call PyImport_ExtendInittab once - PyImport_AppendInittab("{0}", {1}); - } - } REG; -} -''' - - -def mangle(name): - if '.' not in name: - return name - return ''.join('{}{}'.format(len(s), s) for s in name.split('.')) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print >>sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' - print >>sys.stderr, 'Passed: ' + ' '.join(sys.argv) - sys.exit(1) - - with open(sys.argv[2], 'w') as f: - modname = sys.argv[1] - initname = 'PyInit_' + mangle(modname) - code = template.replace('{0}', modname).replace('{1}', initname) - f.write(code) + +namespace { + struct TRegistrar { + inline TRegistrar() { + // TODO Collect all modules and call PyImport_ExtendInittab once + PyImport_AppendInittab("{0}", {1}); + } + } REG; +} +''' + + +def mangle(name): + if '.' not in name: + return name + return ''.join('{}{}'.format(len(s), s) for s in name.split('.')) + +if __name__ == '__main__': + if len(sys.argv) != 3: + print >>sys.stderr, 'Usage: <path/to/gen_py_reg.py> <python_module_name> <output_file>' + print >>sys.stderr, 'Passed: ' + ' '.join(sys.argv) + sys.exit(1) + + with open(sys.argv[2], 'w') as f: + modname = sys.argv[1] + initname = 'PyInit_' + mangle(modname) + code = template.replace('{0}', modname).replace('{1}', initname) + f.write(code) diff --git a/build/scripts/gen_py_protos.py b/build/scripts/gen_py_protos.py index a5355bc03e..08397472f9 100644 --- a/build/scripts/gen_py_protos.py +++ b/build/scripts/gen_py_protos.py @@ -4,42 +4,42 @@ import shutil import subprocess import sys import tempfile -import argparse -import re +import argparse +import re OUT_DIR_ARG = '--python_out=' -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--suffixes", nargs="*", default=[]) - parser.add_argument("protoc_args", nargs=argparse.REMAINDER) - script_args = parser.parse_args() - - args = script_args.protoc_args - - if args[0] == "--": - args = args[1:] - +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--suffixes", nargs="*", default=[]) + parser.add_argument("protoc_args", nargs=argparse.REMAINDER) + script_args = parser.parse_args() + + args = script_args.protoc_args + + if args[0] == "--": + args = args[1:] + out_dir_orig = None out_dir_temp = None - plugin_out_dirs_orig = {} + plugin_out_dirs_orig = {} for i in range(len(args)): if args[i].startswith(OUT_DIR_ARG): assert not out_dir_orig, 'Duplicate "{0}" param'.format(OUT_DIR_ARG) out_dir_orig = args[i][len(OUT_DIR_ARG):] out_dir_temp = tempfile.mkdtemp(dir=out_dir_orig) args[i] = OUT_DIR_ARG + out_dir_temp - continue - - match = re.match(r"^(--(\w+)_out=).*", args[i]) - if match: - plugin_out_dir_arg = match.group(1) - plugin = match.group(2) - assert plugin not in plugin_out_dirs_orig, 'Duplicate "{0}" param'.format(plugin_out_dir_arg) - plugin_out_dirs_orig[plugin] = args[i][len(plugin_out_dir_arg):] - assert plugin_out_dirs_orig[plugin] == out_dir_orig, 'Params "{0}" and "{1}" expected to have the same value'.format(OUT_DIR_ARG, plugin_out_dir_arg) - args[i] = plugin_out_dir_arg + out_dir_temp + continue + + match = re.match(r"^(--(\w+)_out=).*", args[i]) + if match: + plugin_out_dir_arg = match.group(1) + plugin = match.group(2) + assert plugin not in plugin_out_dirs_orig, 'Duplicate "{0}" param'.format(plugin_out_dir_arg) + plugin_out_dirs_orig[plugin] = args[i][len(plugin_out_dir_arg):] + assert plugin_out_dirs_orig[plugin] == out_dir_orig, 'Params "{0}" and "{1}" expected to have the same value'.format(OUT_DIR_ARG, plugin_out_dir_arg) + args[i] = plugin_out_dir_arg + out_dir_temp assert out_dir_temp, 'Param "{0}" not found'.format(OUT_DIR_ARG) @@ -55,13 +55,13 @@ def main(): os.mkdir(d_orig) for f in files: f_orig = f - for suf in script_args.suffixes: - if f.endswith(suf): - f_orig = f[:-len(suf)] + "__int__" + suf + for suf in script_args.suffixes: + if f.endswith(suf): + f_orig = f[:-len(suf)] + "__int__" + suf break os.rename(path.join(root_temp, f), path.join(root_orig, f_orig)) shutil.rmtree(out_dir_temp) if __name__ == '__main__': - main() + main() diff --git a/build/scripts/gen_tasklet_reg.py b/build/scripts/gen_tasklet_reg.py index e854446cb4..5b747c2eca 100644 --- a/build/scripts/gen_tasklet_reg.py +++ b/build/scripts/gen_tasklet_reg.py @@ -1,16 +1,16 @@ -import argparse - +import argparse + TEMPLATE = '''\ {includes}\ #include <tasklet/runtime/lib/{language}_wrapper.h> #include <tasklet/runtime/lib/registry.h> - + static const NTasklet::TRegHelper REG( - "{name}", + "{name}", new NTasklet::{wrapper} -); -''' - +); +''' + WRAPPER = { 'cpp': 'TCppWrapper<{impl}>()', 'js': 'TJsWrapper("{impl}")', @@ -18,34 +18,34 @@ WRAPPER = { 'py': 'TPythonWrapper("{impl}")', 'java': 'TJavaWrapper("{impl}", "{py_wrapper}")', } - - -def parse_args(): - parser = argparse.ArgumentParser() + + +def parse_args(): + parser = argparse.ArgumentParser() parser.add_argument('name') parser.add_argument('output') parser.add_argument('-l', '--lang', choices=WRAPPER, required=True) parser.add_argument('-i', '--impl', required=True) parser.add_argument('-w', '--wrapper', required=False) parser.add_argument('includes', nargs='*') - - return parser.parse_args() - - -if __name__ == '__main__': - args = parse_args() - + + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + includes = ''.join( '#include <{}>\n'.format(include) - for include in args.includes - ) - - code = TEMPLATE.format( - includes=includes, + for include in args.includes + ) + + code = TEMPLATE.format( + includes=includes, language=args.lang, - name=args.name, + name=args.name, wrapper=WRAPPER[args.lang].format(impl=args.impl, py_wrapper=args.wrapper), - ) - + ) + with open(args.output, 'w') as f: - f.write(code) + f.write(code) diff --git a/build/scripts/py_compile.py b/build/scripts/py_compile.py index 2386ba2f68..936dbe8816 100755 --- a/build/scripts/py_compile.py +++ b/build/scripts/py_compile.py @@ -11,7 +11,7 @@ def main(): srcpathx, in_fname, out_fname = sys.argv[1:] srcpath = srcpathx[:-1] - with open(in_fname, 'r') as in_file: + with open(in_fname, 'r') as in_file: source = in_file.read() code = compile(source, srcpath, 'exec', dont_inherit=True) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index 7cc789899a..710165e40d 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -31,7 +31,7 @@ TEST_SRCS( fail_module_cmd.py fetch_from.py fetch_from_external.py - fetch_from_mds.py + fetch_from_mds.py fetch_from_npm.py fetch_from_sandbox.py fetch_resource.py diff --git a/build/sysincl/misc-win.yml b/build/sysincl/misc-win.yml index 7a0f6618e5..e53d4be782 100644 --- a/build/sysincl/misc-win.yml +++ b/build/sysincl/misc-win.yml @@ -58,8 +58,8 @@ - win32.h: contrib/libs/libpq/src/include/port/win32.h - source_filter: "^contrib/libs/curl" - includes: - - nghttp2/nghttp2.h + includes: + - nghttp2/nghttp2.h - includes: - sys/queue.h: contrib/libs/libc_compat/include/windows/sys/queue.h diff --git a/build/ya.conf.json b/build/ya.conf.json index d411725fa0..5f7cc875d6 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -3007,22 +3007,22 @@ } ] }, - "ymakeyndexer": { - "tools": { + "ymakeyndexer": { + "tools": { "ymakeyndexer": { "bottle": "ymakeyndexer", "executable": "ymakeyndexer" } - }, - "platforms": [ + }, + "platforms": [ { "host": { "os": "LINUX" }, "default": true } - ] - }, + ] + }, "transfer-manager": { "tools": { "transfer-manager": { @@ -4392,7 +4392,7 @@ "default": true } ] - }, + }, "horadric": { "tools": { "horadric": { @@ -4471,14 +4471,14 @@ } ] }, - "python3": { - "tools": { + "python3": { + "tools": { "python3": { "bottle": "python3", "executable": "python3" } - }, - "platforms": [ + }, + "platforms": [ { "host": { "os": "LINUX" @@ -4504,7 +4504,7 @@ }, "default": true } - ] + ] }, "run_python_udf": { "tools": { @@ -5621,7 +5621,7 @@ "default": true } ] - }, + }, "skotty": { "tools": { "skotty": { @@ -5650,55 +5650,55 @@ } ] }, - "gcc102-arm-none-eabi": { - "tools": { - "cc": { - "bottle": "gcc102-arm-none-eabi", - "executable": "cc" - }, - "c++": { - "bottle": "gcc102-arm-none-eabi", - "executable": "c++" - }, - "gcov": { - "bottle": "gcc102-arm-none-eabi", - "executable": "gcov" - }, - "c++filt": { - "bottle": "gcc102-arm-none-eabi", - "executable": "c++filt" - }, - "nm": { - "bottle": "gcc102-arm-none-eabi", - "executable": "nm" - }, - "strip": { - "bottle": "gcc102-arm-none-eabi", - "executable": "strip" - } - }, - "platforms": [ - { - "host": { - "os": "LINUX" - }, - "target": { - "os": "NONE", - "arch": "armv8m_cortex_m33" - }, - "default": true - } - ], - "params": { - "type": "gnu", - "match_root": "GCC", - "werror_mode": "compiler_specific", - "c_compiler": "$(GCC)/bin/arm-none-eabi-gcc", - "cxx_compiler": "$(GCC)/bin/arm-none-eabi-g++", - "ar": "$(GCC)/bin/arm-none-eabi-ar", - "objcopy": "$(GCC)/bin/arm-none-eabi-objcopy", - "gcc_version": "10.2.1" - } + "gcc102-arm-none-eabi": { + "tools": { + "cc": { + "bottle": "gcc102-arm-none-eabi", + "executable": "cc" + }, + "c++": { + "bottle": "gcc102-arm-none-eabi", + "executable": "c++" + }, + "gcov": { + "bottle": "gcc102-arm-none-eabi", + "executable": "gcov" + }, + "c++filt": { + "bottle": "gcc102-arm-none-eabi", + "executable": "c++filt" + }, + "nm": { + "bottle": "gcc102-arm-none-eabi", + "executable": "nm" + }, + "strip": { + "bottle": "gcc102-arm-none-eabi", + "executable": "strip" + } + }, + "platforms": [ + { + "host": { + "os": "LINUX" + }, + "target": { + "os": "NONE", + "arch": "armv8m_cortex_m33" + }, + "default": true + } + ], + "params": { + "type": "gnu", + "match_root": "GCC", + "werror_mode": "compiler_specific", + "c_compiler": "$(GCC)/bin/arm-none-eabi-gcc", + "cxx_compiler": "$(GCC)/bin/arm-none-eabi-g++", + "ar": "$(GCC)/bin/arm-none-eabi-ar", + "objcopy": "$(GCC)/bin/arm-none-eabi-objcopy", + "gcc_version": "10.2.1" + } }, "eductl": { "tools": { @@ -6554,17 +6554,17 @@ ] } }, - "ymakeyndexer": { - "formula": { + "ymakeyndexer": { + "formula": { "sandbox_id": 239256113, - "match": "YMAKEYNDEXER" - }, - "executable": { + "match": "YMAKEYNDEXER" + }, + "executable": { "ymakeyndexer": [ "ymakeyndexer" ] - } - }, + } + }, "transfer-manager": { "formula": { "sandbox_id": 135979123, @@ -7237,17 +7237,17 @@ "ptipython" ] } - }, - "python3": { - "formula": { + }, + "python3": { + "formula": { "sandbox_id": 1020522806, - "match": "python3" - }, - "executable": { + "match": "python3" + }, + "executable": { "python3": [ "python3" ] - } + } }, "run_python_udf": { "formula": { @@ -7906,7 +7906,7 @@ "wall-e" ] } - }, + }, "skotty": { "formula": { "sandbox_id": 1160938937, @@ -7918,43 +7918,43 @@ ] } }, - "gcc102-arm-none-eabi": { - "formula": { - "sandbox_id": 1020757602, - "match": "GNU ARM" - }, - "executable": { - "c++": [ - "bin", - "arm-none-eabi", - "arm-none-eabi-g++" - ], - "cc": [ - "bin", - "arm-none-eabi", - "arm-none-eabi-gcc" - ], - "gcov": [ - "bin", - "arm-none-eabi", - "arm-none-eabi-gcov" - ], - "c++filt": [ - "bin", - "arm-none-eabi", - "arm-none-eabi-c++filt" - ], - "nm": [ - "bin", - "arm-none-eabi", - "arm-none-eabi-nm" - ], - "strip": [ - "bin", - "arm-none-eabi", - "arm-none-eabi-strip" - ] - } + "gcc102-arm-none-eabi": { + "formula": { + "sandbox_id": 1020757602, + "match": "GNU ARM" + }, + "executable": { + "c++": [ + "bin", + "arm-none-eabi", + "arm-none-eabi-g++" + ], + "cc": [ + "bin", + "arm-none-eabi", + "arm-none-eabi-gcc" + ], + "gcov": [ + "bin", + "arm-none-eabi", + "arm-none-eabi-gcov" + ], + "c++filt": [ + "bin", + "arm-none-eabi", + "arm-none-eabi-c++filt" + ], + "nm": [ + "bin", + "arm-none-eabi", + "arm-none-eabi-nm" + ], + "strip": [ + "bin", + "arm-none-eabi", + "arm-none-eabi-strip" + ] + } }, "eductl": { "formula": { diff --git a/build/ymake.core.conf b/build/ymake.core.conf index a89f4af393..081833998b 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -300,7 +300,7 @@ LUA_TOOL=${tool:"tools/lua"} ENUM_PARSER_TOOL=${tool:"tools/enum_parser/enum_parser"} # tag:python-specific tag:cython-specific -CYTHON_SCRIPT=${input:"${ARCADIA_ROOT}/contrib/tools/cython/cython.py"} +CYTHON_SCRIPT=${input:"${ARCADIA_ROOT}/contrib/tools/cython/cython.py"} RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT CYTHON_OUTPUT_INCLUDES=\ ${output_include;hide:"contrib/libs/python/Include/compile.h"} \ @@ -493,26 +493,26 @@ GEN_PY_PROTOS=$YMAKE_PYTHON ${input:"build/scripts/gen_py_protos.py"} # tag:proto tag:cpp-specific PROTO_HEADER_EXTS=.pb.h CPP_PROTO_CMDLINE= -CPP_PROTO_OPTS= +CPP_PROTO_OPTS= CPP_PROTO_OUTS= -CPP_PROTO_SUFFIXES=.pb.h .pb.cc +CPP_PROTO_SUFFIXES=.pb.h .pb.cc CPP_PROTO_PLUGINS= - + # tag:proto tag:cpp-specific CPP_EV_OPTS=--plugin=protoc-gen-event2cpp=${tool:"tools/event2cpp"} --event2cpp_out=$ARCADIA_BUILD_ROOT -I=$ARCADIA_ROOT/library/cpp/eventlog CPP_EV_OUTS= # tag:proto tag:python-specific -PY_PROTO_OPTS= -PY_PROTO_OUTS= -PY_PROTO_OUTS_INTERNAL= -PY_PROTO_DEPS= -PY_PROTO_SUFFIXES=_pb2.py -PY_EVLOG_SUFFIXES=_ev_pb2.py - +PY_PROTO_OPTS= +PY_PROTO_OUTS= +PY_PROTO_OUTS_INTERNAL= +PY_PROTO_DEPS= +PY_PROTO_SUFFIXES=_pb2.py +PY_EVLOG_SUFFIXES=_ev_pb2.py + # tag:proto tag:java-specific -JAVA_PROTO_ARGS= - +JAVA_PROTO_ARGS= + # tag:proto tag:python-specific OPTIMIZE_PY_PROTOS_FLAG=no @@ -607,16 +607,16 @@ macro NO_OPTIMIZE_PY_PROTOS() { macro _PROTO_PLUGIN_ARGS_BASE(Name, Tool, OutParm...) { .CMD=--plugin=protoc-gen-${Name}=\${tool:"$Tool"} --${Name}_out=$OutParm$ARCADIA_BUILD_ROOT/\$PROTO_NAMESPACE .SEM=ignored -} - +} + # tag:proto tag:python-specific macro _ADD_PY_PROTO_OUT(Suf) { SET_APPEND(PY_PROTO_OUTS \${output;hide;noauto;norel;nopath;noext;suf=$Suf:File}) - SET_APPEND(PY_PROTO_OUTS_INTERNAL \${output;hide;noauto;norel;nopath;noext;suf=__int__$Suf:File} \${hide;kv:"ext_out_name_for_\${nopath;noext;suf=__int__$Suf:File} \${nopath;noext;suf=$Suf:File}"}) - # XXX fix variable expansion in plugins - SET(PY_PROTO_SUFFIXES $PY_PROTO_SUFFIXES $Suf) -} - + SET_APPEND(PY_PROTO_OUTS_INTERNAL \${output;hide;noauto;norel;nopath;noext;suf=__int__$Suf:File} \${hide;kv:"ext_out_name_for_\${nopath;noext;suf=__int__$Suf:File} \${nopath;noext;suf=$Suf:File}"}) + # XXX fix variable expansion in plugins + SET(PY_PROTO_SUFFIXES $PY_PROTO_SUFFIXES $Suf) +} + # tag:proto tag:python-specific ### @usage: PY_PROTO_PLUGIN(Name Ext Tool DEPS <Dependencies>) ### @@ -846,7 +846,7 @@ macro YP_PROTO_YSON(OUT_OPTS[], Files...) { # tag:proto macro _CPP_PROTO_CMD(File) { - .CMD=$CPP_PROTO_CMDLINE $CPP_PROTO_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} + .CMD=$CPP_PROTO_CMDLINE $CPP_PROTO_OPTS $CPP_PROTO_OUTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} .SEM=target_proto_mesages PRIVATE ${input:File} ${output;hide;suf=.pb.o:File} ${output;main;hide;norel;nopath;noext:File.pb.h} ${hide;tool:"contrib/tools/protoc/bin"} && set_global_flags COMMON_PROTOC_FLAGS -I=$ARCADIA_ROOT -I=$ARCADIA_BUILD_ROOT && modules_required protobuf.cmake .PEERDIR=contrib/libs/protobuf } @@ -1021,14 +1021,14 @@ SANITIZER_CFLAGS= macro GRPC() { # C++ CPP_PROTO_PLUGIN2(grpc_cpp contrib/tools/protoc/plugins/grpc_cpp .grpc.pb.cc .grpc.pb.h DEPS contrib/libs/grpc) - - # Python + + # Python PY_PROTO_PLUGIN(grpc_py _pb2_grpc.py contrib/tools/protoc/plugins/grpc_python DEPS contrib/libs/grpc/python contrib/libs/grpc) - - # Java + + # Java JAVA_PROTO_PLUGIN(grpc_java contrib/tools/protoc/plugins/grpc_java DEPS $JAVA_GRPC_STUB $JAVA_GRPC_PROTOBUF) } - + # tag:fbs FLATC_FLAGS_VALUE= ### @usage: FLATC_FLAGS(flags...) @@ -1569,7 +1569,7 @@ macro LINK_EXEC_DYN_LIB_IMPL(WHOLE_ARCHIVE_PEERS...) { ### ### The base of all linkable modules: programs, DLLs etc. Describes common linking logic. module _LINK_UNIT: _BASE_UNIT { - .EXTS=.o .obj .supp .tidyjson .ld + .EXTS=.o .obj .supp .tidyjson .ld .CMD=LINK_EXE .ALLOWED=EXTRALIBS OBJADDE_GLOBAL RESOURCE_FILES .NODE_TYPE=Program @@ -1745,8 +1745,8 @@ multimodule PY3_PROGRAM { ENABLE(IGNORE_PY_MAIN) _REQUIRE_EXPLICIT_LICENSE() } -} - +} + # tag:ycr-specific macro _YCR_GENERATE_CONFIGS_INTL(Package, App, Configs...) { .CMD=${tool:"maps/infra/yacare/scripts/make_configs"} $Package $App ${cwd:BINDIR} ${output;hide;pre=debian/$Package/:Configs} @@ -2356,7 +2356,7 @@ module _LIBRARY: _BASE_UNIT { .CMD=LINK_LIB .NODE_TYPE=Library .PEERDIR_POLICY=as_include - .EXTS=.o .obj .a .mf .supp .tidyjson .ld + .EXTS=.o .obj .a .mf .supp .tidyjson .ld .ALLOWED=GRPC USE_SKIFF EXTRALIBS OBJADDE_GLOBAL RESOURCE_FILES .GLOBAL=USER_CFLAGS USER_CXXFLAGS USER_CONLYFLAGS LDFLAGS SRCS _WHOLE_ARCHIVE_LIBS_VALUE RPATH .RESTRICTED=ALLOCATOR SIZE TAG DATA TEST_DATA DEPENDS FORK_TESTS FORK_SUBTESTS SPLIT_FACTOR TEST_CWD RUN TIMEOUT SPLIT_DWARF @@ -4062,12 +4062,12 @@ module _PY3_LIBRARY: _LIBRARY { when ($NO_PYTHON_INCLS != "yes") { PEERDIR+=contrib/libs/python } - when ($MSVC == "yes" || $CYGWIN == "yes") { - MODULE_PREFIX=py - } - otherwise { - MODULE_PREFIX=libpy - } + when ($MSVC == "yes" || $CYGWIN == "yes") { + MODULE_PREFIX=py + } + otherwise { + MODULE_PREFIX=libpy + } .ALLOWED=FORK_TESTS .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER .RESTRICTED=PYTHON3_ADDINCL USE_PYTHON3 PYTHON2_ADDINCL USE_PYTHON2 OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS @@ -4082,8 +4082,8 @@ module _PY3_LIBRARY: _LIBRARY { PY_PROTO_MYPY_PLUGIN_INTERNAL=$PY_PROTO_MYPY_PLUGIN_BASE ${output;hide;noauto;norel;nopath;noext;suf=__int___pb2.pyi:File} ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__int___pb2.pyi:File} ${nopath;noext;suf=_pb2.pyi:File}"}) } SET(MODULE_LANG PY3) -} - +} + # tag:python-specific ### @usage: PY3_LIBRARY() ### @@ -4171,11 +4171,11 @@ otherwise { module _BASE_PY3_PROGRAM: _BASE_PROGRAM { .CMD=PY3_PROGRAM_LINK_EXE _ARCADIA_PYTHON3_ADDINCL() - PEERDIR(library/python/runtime_py3/main) + PEERDIR(library/python/runtime_py3/main) when ($PYTHON_SQLITE3 != "no") { PEERDIR += contrib/tools/python3/src/Modules/_sqlite } - + when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") { NO_CHECK_IMPORTS_FOR_VALUE= } @@ -4188,17 +4188,17 @@ module _BASE_PY3_PROGRAM: _BASE_PROGRAM { _MY_ALLOCATOR=J } ALLOCATOR($_MY_ALLOCATOR) - STRIP() - + STRIP() + when ($NO_PYTHON_INCLS != "yes") { PEERDIR+=contrib/libs/python } when ($BUILD_TYPE == "DEBUG") { NO_STRIP=yes } - when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") { - NO_STRIP=yes - } + when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") { + NO_STRIP=yes + } when ($PYTHON_COVERAGE == "yes") { PEERDIR+=library/python/coverage } @@ -4208,8 +4208,8 @@ module _BASE_PY3_PROGRAM: _BASE_PROGRAM { .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER .RESTRICTED=PYTHON3_ADDINCL USE_PYTHON3 PYTHON2_ADDINCL USE_PYTHON2 SET(MODULE_LANG PY3) -} - +} + # tag:python-specific ### @usage: PY3_PROGRAM_BIN([progname]) ### Use instead of PY3_PROGRAM only if ya.make with PY3_PROGRAM() included in another ya.make @@ -5198,8 +5198,8 @@ macro USE_PYTHON2() { _ARCADIA_PYTHON_ADDINCL() SET(PEERDIR_TAGS PY2 PY2_NATIVE CPP_PROTO CPP_FBS YQL_UDF_STATIC __EMPTY__ DLL_LIB) PEERDIR(contrib/libs/python) -} - +} + # tag:python-specific ### @usage: USE_PYTHON3() ### @@ -5211,16 +5211,16 @@ macro USE_PYTHON2() { ### If you'd like to use #include <Python.h> with both Python2 and Python3 convert your LIBRARY to PY23_LIBRARY. ### ### @see: [PY2_LIBRARY](#module_PY2_LIBRARY), [PY3_LIBRARY](#module_PY3_LIBRARY), [PY23_LIBRARY](#multimodule_PY23_LIBRARY) -macro USE_PYTHON3() { +macro USE_PYTHON3() { _ARCADIA_PYTHON3_ADDINCL() SET(PEERDIR_TAGS PY3 PY3_BIN_LIB PY3TEST_LIBRARY PY3_NATIVE CPP_PROTO CPP_FBS YQL_UDF_STATIC __EMPTY__ DLL_LIB) PEERDIR(contrib/libs/python) - + when ($USE_ARCADIA_PYTHON == "yes") { PEERDIR+=library/python/runtime_py3 } -} - +} + # tag:python-specific when ($PERL_DEFAULT_PEERDIR == "yes") { when (($PERLXS == "yes") || ($PERLXSCPP == "yes") || ($USE_PERL == "yes")) { @@ -5382,7 +5382,7 @@ macro SPLIT_DWARF() { SPLIT_DWARF_OUTPUT=${output;tobindir;pre=$MODULE_PREFIX;suf=$MODULE_SUFFIX.debug:REALPRJNAME} when ($SPLIT_DWARF_VALUE == "yes" && $NO_SPLIT_DWARF != "yes" && $NO_DEBUGINFO != "yes" && $HOST_OS_LINUX == "yes" && $TARGET_PLATFORM == "LINUX") { - DWARF_COMMAND+= \ + DWARF_COMMAND+= \ $OBJCOPY_TOOL --only-keep-debug $TARGET $SPLIT_DWARF_OUTPUT && \ $STRIP_TOOL --strip-debug $TARGET && \ $OBJCOPY_TOOL --remove-section=.gnu_debuglink --add-gnu-debuglink $SPLIT_DWARF_OUTPUT $TARGET @@ -5745,10 +5745,10 @@ macro _SRC("s", SRC, SRCFLAGS...) { } # tag:src-processing -macro _SRC("s79", SRC, SRCFLAGS...) { - .CMD=$_SRC(S, $SRC $SRCFLAGS -x assembler-with-cpp) -} - +macro _SRC("s79", SRC, SRCFLAGS...) { + .CMD=$_SRC(S, $SRC $SRCFLAGS -x assembler-with-cpp) +} + # tag:src-processing macro _SRC("mm", SRC, SRCFLAGS...) { .CMD=$C_COMPILER $C_FLAGS_PLATFORM -x objective-c++ -fobjc-arc -fobjc-abi-version=2 -c -o ${output:SRC.o} ${input:SRC} $CXXFLAGS ${pre=-I:_C__INCLUDE} ${SRCFLAGS} ${requirements;hide:CC_REQUIREMENTS} ${kv;hide:"p CC"} ${kv;hide:"pc light-green"} $TOOLCHAIN_ENV @@ -6370,11 +6370,11 @@ macro _PY_REGISTER(Func) { ### Register Python 3.x module in internal resource file system. Arcadia Python 3.x importer will be retrieve these on import directive ### ### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/pysrcs/#makrospyregister -macro _PY3_REGISTER(Func) { +macro _PY3_REGISTER(Func) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/gen_py3_reg.py"} $Func ${output;noauto:Func.reg3.cpp} ${requirements;hide:PY_REQUIREMENTS} ${kv;hide:"p PY"} ${kv;hide:"pc yellow"} SRCS(GLOBAL $Func.reg3.cpp) -} - +} + # tag:python-processing tag:internal ### @usage: _PY_COMPILE_BYTECODE(SrcX Src) # internal ### @@ -6393,8 +6393,8 @@ macro _PY_COMPILE_BYTECODE(SrcX, Src, Dst) { ### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/pysrcs/#makrospyregister macro _PY3_COMPILE_BYTECODE(SrcX, Src, Dst) { .CMD=${env:"PYTHONHASHSEED=0"} ${tool:"contrib/tools/python3/pycc"} $SrcX ${input:Src} ${output;noauto:Dst.yapyc3} ${requirements;hide:PY_REQUIREMENTS} ${kv;hide:"p PY"} ${kv;hide:"pc yellow"} -} - +} + ### @usage: ARCHIVE_ASM(NAME archive_name files...) ### ### Similar to the macro ARCHIVE, but: @@ -7076,8 +7076,8 @@ macro FROM_SANDBOX(Id, OUT{output}[], OUT_NOAUTO{output}[], OUTPUT_INCLUDES[], F macro FROM_MDS(Key, OUT{output}[], OUT_NOAUTO{output}[], OUTPUT_INCLUDES[], FILE?"--copy-to-dir":"--untar-to", PREFIX=".", RENAME[], EXECUTABLE?"--executable":"") { .CMD=${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/fetch_from_mds.py"} --key $Key $FILE $PREFIX ${pre=--rename :RENAME} $EXECUTABLE -- $OUT $OUT_NOAUTO ${input;hide:"build/scripts/fetch_from.py"} ${output_include;hide:OUTPUT_INCLUDES} ${output;hide:OUT} ${output;noauto;hide:OUT_NOAUTO} ${requirements;hide:"network:full"} ${kv;hide:"p MD"} ${kv;hide:"pc yellow"} ${kv;hide:"show_out"} ADD_CHECK(check.mds $Key) -} - +} + # tag:internal ### @usage: _FROM_EXTERNAL(ExtFile [AUTOUPDATED script] [RENAME <resource files>] OUT_[NOAUTO] <output files> [EXECUTABLE]) #internal ###requirements;hide @@ -7382,11 +7382,11 @@ ANDROID_SDK_ROOT=${ANDROID_SDK_RESOURCE_GLOBAL}/android_sdk macro TASKLET() { PEERDIR(tasklet/api) - - # CPP + + # CPP CPP_PROTO_PLUGIN(tasklet_cpp tasklet/gen/cpp .tasklet.h) - - # Python + + # Python PY_PROTO_PLUGIN2(tasklet_py _tasklet.py _sbtask.py tasklet/gen/python DEPS tasklet/domain/sandbox tasklet/runtime sandbox/sdk2) #Java @@ -7399,7 +7399,7 @@ TASKLET_REG_INCLUDES= \ ${output_include;hide:"tasklet/runtime/lib/py_wrapper.h"} \ ${output_include;hide:"tasklet/runtime/lib/js_wrapper.h"} \ ${output_include;hide:"tasklet/runtime/lib/registry.h"} - + macro TASKLET_REG(Name, Lang, Impl, Includes...) { PEERDIR(tasklet/domain sandbox/bin sandbox/taskbox/worker) @@ -7411,8 +7411,8 @@ macro TASKLET_REG(Name, Lang, Impl, Includes...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/gen_tasklet_reg.py"} $Name -l $Lang -i $Impl ${output;noauto:Name.task.cpp} $Includes ${output_include;hide:Includes} $TASKLET_REG_INCLUDES ${kv;hide:"p TT"} ${kv;hide:"pc yellow"} SRCS(GLOBAL $Name.task.cpp) -} - +} + # TEMPORARY METHOD FOR EXTENDED REGISTRY SETUP # NOT COMPLETE macro TASKLET_REG_EXT(Name, Lang, Impl, Wrapper, Includes...) { @@ -7540,7 +7540,7 @@ multimodule PROTO_LIBRARY { } module PY3_PROTO: PY3_LIBRARY { - .ALIASES=SRCS=PY_SRCS + .ALIASES=SRCS=PY_SRCS .ALLOWED=OPTIMIZE_PY_PROTOS NO_OPTIMIZE_PY_PROTOS .PEERDIRSELF=CPP_PROTO .SEM=ignored @@ -8828,7 +8828,7 @@ multimodule PY23_NATIVE_LIBRARY { module PY3: LIBRARY { .RESTRICTED=PY_SRCS USE_PYTHON2 USE_PYTHON3 .ALIASES=PYTHON2_ADDINCL=PYTHON3_ADDINCL - PYTHON3_ADDINCL() + PYTHON3_ADDINCL() SET(MODULE_LANG PY3) when ($MSVC == "yes" || $CYGWIN == "yes") { MODULE_PREFIX=py3c @@ -9035,17 +9035,17 @@ multimodule PY23_TEST { RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT } } - + # tag:windows-specific -WINDOWS_MANIFEST= -macro WINDOWS_MANIFEST(Manifest) { - SET(WINDOWS_MANIFEST $Manifest) -} - +WINDOWS_MANIFEST= +macro WINDOWS_MANIFEST(Manifest) { + SET(WINDOWS_MANIFEST $Manifest) +} + # tag:windows-specific -when ($MSVC == "yes" && $WINDOWS_MANIFEST) { - LDFLAGS+=/MANIFEST:EMBED /MANIFESTINPUT:${input:WINDOWS_MANIFEST} -} +when ($MSVC == "yes" && $WINDOWS_MANIFEST) { + LDFLAGS+=/MANIFEST:EMBED /MANIFESTINPUT:${input:WINDOWS_MANIFEST} +} # tag:cpp-specific _VISIBILITY_LEVEL= @@ -9073,18 +9073,18 @@ when ($_VISIBILITY_LEVEL != "" && $IGNORE_VISIBILITY != "yes") { } } CFLAGS+=$_CFLAGS_VISIBILITY - + # tag:cpp-specific -macro SDBUS_CPP_ADAPTOR(File) { - .CMD=${tool:"contrib/libs/sdbus-cpp/tools/xml2cpp-codegen"} --adaptor=${output;nopath;noext:File.adaptor.h} ${input:File} - .PEERDIR=contrib/libs/sdbus-cpp -} - +macro SDBUS_CPP_ADAPTOR(File) { + .CMD=${tool:"contrib/libs/sdbus-cpp/tools/xml2cpp-codegen"} --adaptor=${output;nopath;noext:File.adaptor.h} ${input:File} + .PEERDIR=contrib/libs/sdbus-cpp +} + # tag:cpp-specific -macro SDBUS_CPP_PROXY(File) { - .CMD=${tool:"contrib/libs/sdbus-cpp/tools/xml2cpp-codegen"} --proxy=${output;nopath;noext:File.proxy.h} ${input:File} - .PEERDIR=contrib/libs/sdbus-cpp -} +macro SDBUS_CPP_PROXY(File) { + .CMD=${tool:"contrib/libs/sdbus-cpp/tools/xml2cpp-codegen"} --proxy=${output;nopath;noext:File.proxy.h} ${input:File} + .PEERDIR=contrib/libs/sdbus-cpp +} # tag:python-specific @@ -9181,7 +9181,7 @@ module CUSTOM_BUILD_LIBRARY: LIBRARY { macro ORIGINAL_SOURCE(Source) { ENABLE(UNUSED_MACRO) } - + CMAKE_PACKAGE= CMAKE_PACKAGE_COMPONENT= CMAKE_PACKAGE_TARGET= @@ -9228,19 +9228,19 @@ when ($WITH_CREDITS) { GENERATE_MF=$GENERATE_MF_CMD } -macro _MCU_CONVERT(Bin) { - .CMD=${cwd:BINDIR} $OBJCOPY_TOOL -O ihex $Bin ${output;noext:Bin.hex} && ${cwd:BINDIR} $OBJCOPY_TOOL -O srec $Bin ${output;noext:Bin.s37} -} - -MCU_LINK_EXE=$LINK_EXE && $_MCU_CONVERT($MODULE_PREFIX$REALPRJNAME) - -### @usage: MCU_PROGRAM([progname]) -### -### Program module for microcontrollers. Converts binary to Intel HEX and Motorola S-record file formats. -### If name is not specified it will be generated from the name of the containing project directory. -module MCU_PROGRAM: PROGRAM { - .CMD=MCU_LINK_EXE -} +macro _MCU_CONVERT(Bin) { + .CMD=${cwd:BINDIR} $OBJCOPY_TOOL -O ihex $Bin ${output;noext:Bin.hex} && ${cwd:BINDIR} $OBJCOPY_TOOL -O srec $Bin ${output;noext:Bin.s37} +} + +MCU_LINK_EXE=$LINK_EXE && $_MCU_CONVERT($MODULE_PREFIX$REALPRJNAME) + +### @usage: MCU_PROGRAM([progname]) +### +### Program module for microcontrollers. Converts binary to Intel HEX and Motorola S-record file formats. +### If name is not specified it will be generated from the name of the containing project directory. +module MCU_PROGRAM: PROGRAM { + .CMD=MCU_LINK_EXE +} # tag:deprecated ### @usage: METAQUERYFILES(filenames...) #deprecated diff --git a/build/ymake_conf.py b/build/ymake_conf.py index f7626f9687..30219eb85e 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -56,9 +56,9 @@ class Platform(object): self.is_armv7 = self.arch in ('armv7', 'armv7a', 'armv7a_neon', 'arm', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53') self.is_armv8 = self.arch in ('armv8', 'armv8a', 'arm64', 'aarch64', 'armv8a_cortex_a35', 'armv8a_cortex_a53') - self.is_armv8m = self.arch in ('armv8m_cortex_m33',) + self.is_armv8m = self.arch in ('armv8m_cortex_m33',) self.is_arm64 = self.arch in ('arm64',) - self.is_arm = self.is_armv7 or self.is_armv8 or self.is_armv8m + self.is_arm = self.is_armv7 or self.is_armv8 or self.is_armv8m self.is_armv7_neon = self.arch in ('armv7a_neon', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53') self.is_armv7hf = self.arch in ('armv7ahf_cortex_a35', 'armv7ahf_cortex_a53') @@ -72,13 +72,13 @@ class Platform(object): self.is_cortex_a9 = self.arch in ('armv7a_cortex_a9',) self.is_cortex_a35 = self.arch in ('armv7ahf_cortex_a35', 'armv8a_cortex_a35') self.is_cortex_a53 = self.arch in ('armv7ahf_cortex_a53', 'armv8a_cortex_a53') - self.is_cortex_m33 = self.arch in ('armv8m_cortex_m33',) + self.is_cortex_m33 = self.arch in ('armv8m_cortex_m33',) self.is_power8le = self.arch == 'ppc64le' self.is_power9le = self.arch == 'power9le' self.is_powerpc = self.is_power8le or self.is_power9le - self.is_32_bit = self.is_x86 or self.is_armv7 or self.is_armv8m + self.is_32_bit = self.is_x86 or self.is_armv7 or self.is_armv8m self.is_64_bit = self.is_x86_64 or self.is_armv8 or self.is_powerpc assert self.is_32_bit or self.is_64_bit @@ -113,8 +113,8 @@ class Platform(object): self.is_cygwin = self.os == 'cygwin' self.is_yocto = self.os == 'yocto' - self.is_none = self.os == 'none' - + self.is_none = self.os == 'none' + self.is_posix = self.is_linux or self.is_apple or self.is_android or self.is_cygwin or self.is_yocto @staticmethod @@ -152,7 +152,7 @@ class Platform(object): (self.is_armv7, 'ARCH_ARM7'), (self.is_armv7_neon, 'ARCH_ARM7_NEON'), (self.is_armv8, 'ARCH_ARM64'), - (self.is_armv8m, 'ARCH_ARM8M'), + (self.is_armv8m, 'ARCH_ARM8M'), (self.is_arm, 'ARCH_ARM'), (self.is_linux_armv8 or self.is_macos_arm64, 'ARCH_AARCH64'), (self.is_powerpc, 'ARCH_PPC64LE'), @@ -932,7 +932,7 @@ class ToolchainOptions(object): if detector: self.type = detector.type - self.from_arcadia = False + self.from_arcadia = False self.c_compiler = detector.c_compiler self.cxx_compiler = detector.cxx_compiler @@ -941,7 +941,7 @@ class ToolchainOptions(object): else: self.type = self.params['type'] - self.from_arcadia = True + self.from_arcadia = True self.c_compiler = self.params['c_compiler'] self.cxx_compiler = self.params['cxx_compiler'] @@ -991,14 +991,14 @@ class ToolchainOptions(object): def is_clang(self): return self.type in ('clang', 'xcode') - @property + @property def is_xcode(self): return self.type == 'xcode' @property - def is_from_arcadia(self): - return self.from_arcadia - + def is_from_arcadia(self): + return self.from_arcadia + def get_env(self, convert_list=None): convert_list = convert_list or (lambda x: x) r = {} @@ -1200,13 +1200,13 @@ class GnuToolchain(Toolchain): elif target.is_cortex_a53: self.c_flags_platform.append('-mcpu=cortex-a53') - elif target.is_cortex_m33: - self.c_flags_platform.append('-mcpu=cortex-m33 -mfpu=fpv5-sp-d16') - + elif target.is_cortex_m33: + self.c_flags_platform.append('-mcpu=cortex-m33 -mfpu=fpv5-sp-d16') + elif target.is_armv7_neon: self.c_flags_platform.append('-mfpu=neon') - if (target.is_armv7 or target.is_armv8m) and build.is_size_optimized: + if (target.is_armv7 or target.is_armv8m) and build.is_size_optimized: # Enable ARM Thumb2 variable-length instruction encoding # to reduce code size self.c_flags_platform.append('-mthumb') @@ -1236,29 +1236,29 @@ class GnuToolchain(Toolchain): if target.is_ios: self.c_flags_platform.append('-D__IOS__=1') - if self.tc.is_from_arcadia: - if target.is_apple: + if self.tc.is_from_arcadia: + if target.is_apple: if target.is_ios: - self.setup_sdk(project='build/platform/ios_sdk', var='${IOS_SDK_ROOT_RESOURCE_GLOBAL}') + self.setup_sdk(project='build/platform/ios_sdk', var='${IOS_SDK_ROOT_RESOURCE_GLOBAL}') self.platform_projects.append('build/platform/macos_system_stl') - if target.is_macos: - self.setup_sdk(project='build/platform/macos_sdk', var='${MACOS_SDK_RESOURCE_GLOBAL}') + if target.is_macos: + self.setup_sdk(project='build/platform/macos_sdk', var='${MACOS_SDK_RESOURCE_GLOBAL}') self.platform_projects.append('build/platform/macos_system_stl') if not self.tc.inplace_tools: self.setup_tools(project='build/platform/cctools', var='${CCTOOLS_ROOT_RESOURCE_GLOBAL}', bin='bin', ldlibs=None) - if target.is_linux: - if not tc.os_sdk_local: - self.setup_sdk(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL') + if target.is_linux: + if not tc.os_sdk_local: + self.setup_sdk(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL') - if target.is_x86_64: + if target.is_x86_64: if host.is_linux and not self.tc.is_gcc: - self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/lib/x86_64-linux-gnu') - elif host.is_macos: - self.setup_tools(project='build/platform/binutils', var='$BINUTILS_ROOT_RESOURCE_GLOBAL', bin='x86_64-linux-gnu/bin', ldlibs=None) + self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/lib/x86_64-linux-gnu') + elif host.is_macos: + self.setup_tools(project='build/platform/binutils', var='$BINUTILS_ROOT_RESOURCE_GLOBAL', bin='x86_64-linux-gnu/bin', ldlibs=None) elif target.is_powerpc: - self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/x86_64-linux-gnu/powerpc64le-linux-gnu/lib') + self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/x86_64-linux-gnu/powerpc64le-linux-gnu/lib') elif target.is_armv8: self.setup_tools(project='build/platform/linux_sdk', var='$OS_SDK_ROOT_RESOURCE_GLOBAL', bin='usr/bin', ldlibs='usr/lib/x86_64-linux-gnu') @@ -1816,13 +1816,13 @@ class LD(Linker): if target.is_apple: # Use libtool. cctools ar does not understand -M needed for archive merging self.ar = '${CCTOOLS_ROOT_RESOURCE_GLOBAL}/bin/libtool' - elif self.tc.is_from_arcadia: + elif self.tc.is_from_arcadia: if self.tc.is_clang: self.ar = '{}/bin/llvm-ar'.format(self.tc.name_marker) if self.tc.is_gcc: self.ar = '{}/gcc/bin/gcc-ar'.format(self.tc.name_marker) else: - self.ar = 'ar' + self.ar = 'ar' self.ar_type = 'GNU_AR' self.llvm_ar_format = 'None' @@ -1907,7 +1907,7 @@ class LD(Linker): self.ld_export_dynamic_flag = '-rdynamic' self.use_stdlib = '-nodefaultlibs' - if target.is_linux or target.is_android or target.is_cygwin or target.is_none: + if target.is_linux or target.is_android or target.is_cygwin or target.is_none: self.start_group = '-Wl,--start-group' self.end_group = '-Wl,--end-group' self.whole_archive = '-Wl,--whole-archive' @@ -2022,8 +2022,8 @@ class LD(Linker): EXPORTS_VALUE= }''') - emit('LINKER_SCRIPT_VALUE', '${ext=.ld;pre=-T:SRCS_GLOBAL}') - + emit('LINKER_SCRIPT_VALUE', '${ext=.ld;pre=-T:SRCS_GLOBAL}') + linker_time_trace_requested = is_positive('TIME_TRACE') or is_positive('LINKER_TIME_TRACE') linker_supports_time_trace = self.type == Linker.LLD # XXX: Should really check the linker version if we had one if linker_time_trace_requested and linker_supports_time_trace: @@ -2038,7 +2038,7 @@ class LD(Linker): exe_flags = [ '$C_FLAGS_PLATFORM', '$BEFORE_PEERS', self.start_group, '${rootrel:PEERS}', self.end_group, '$AFTER_PEERS', - '$EXPORTS_VALUE $LINKER_SCRIPT_VALUE $LDFLAGS $LDFLAGS_GLOBAL $OBJADDE $OBJADDE_LIB', + '$EXPORTS_VALUE $LINKER_SCRIPT_VALUE $LDFLAGS $LDFLAGS_GLOBAL $OBJADDE $OBJADDE_LIB', '$C_LIBRARY_PATH $C_SYSTEM_LIBRARIES_INTERCEPT $C_SYSTEM_LIBRARIES $STRIP_FLAG $DCE_FLAG $ICF_FLAG $LINKER_TIME_TRACE_FLAG'] arch_flag = '--arch={arch}'.format(arch=self.target.os_compat) @@ -2158,7 +2158,7 @@ class LD(Linker): def emit_link_fat_obj(cmd_name, need_wa_option, *extended_flags): prefix = ['$GENERATE_MF && $GENERATE_VCS_C_INFO_NODEP &&', '$YMAKE_PYTHON ${input:"build/scripts/link_fat_obj.py"} --build-root $ARCADIA_BUILD_ROOT'] - globals_libs = srcs_globals if need_wa_option else '${rootrel;ext=.a:SRCS_GLOBAL} ${rootrel;ext=.o:SRCS_GLOBAL}' + globals_libs = srcs_globals if need_wa_option else '${rootrel;ext=.a:SRCS_GLOBAL} ${rootrel;ext=.o:SRCS_GLOBAL}' suffix = [arch_flag, '-Ya,input $AUTO_INPUT $VCS_C_OBJ -Ya,global_srcs', globals_libs, '-Ya,peers $PEERS', '-Ya,linker $CXX_COMPILER $LDFLAGS_GLOBAL $C_FLAGS_PLATFORM', self.ld_sdk, '-Ya,archiver', archiver, @@ -2809,7 +2809,7 @@ class MSVCLinker(MSVC, Linker): emit('LINK_EXEC_DYN_LIB', '$LINK_EXEC_DYN_LIB_IMPL($_WHOLE_ARCHIVE_PEERS_VALUE)') emit('LINK_GLOBAL_FAT_OBJECT', '${TOOLCHAIN_ENV} ${cwd:ARCADIA_BUILD_ROOT} ${LIB_WRAPPER} ${LINK_LIB_CMD} /OUT:${qe;rootrel:TARGET} \ - --ya-start-command-file ${qe;rootrel;ext=.lib:SRCS_GLOBAL} ${qe;rootrel;ext=.obj:SRCS_GLOBAL} ${qe;rootrel:AUTO_INPUT} $LINK_LIB_FLAGS --ya-end-command-file') + --ya-start-command-file ${qe;rootrel;ext=.lib:SRCS_GLOBAL} ${qe;rootrel;ext=.obj:SRCS_GLOBAL} ${qe;rootrel:AUTO_INPUT} $LINK_LIB_FLAGS --ya-end-command-file') emit('LINK_PEERS_FAT_OBJECT', '${TOOLCHAIN_ENV} ${cwd:ARCADIA_BUILD_ROOT} ${LIB_WRAPPER} ${LINK_LIB_CMD} /OUT:${qe;rootrel;output:REALPRJNAME.lib} \ --ya-start-command-file ${qe;rootrel:PEERS} $LINK_LIB_FLAGS --ya-end-command-file') emit('LINK_FAT_OBJECT', '${GENERATE_MF} && $GENERATE_VCS_C_INFO_NODEP && $LINK_GLOBAL_FAT_OBJECT && $LINK_PEERS_FAT_OBJECT ${kv;hide:"p LD"} ${requirements;hide:LD_REQUIREMENTS} ${kv;hide:"pc light-blue"} ${kv;hide:"show_out"}') # noqa E501 |