diff options
| author | rnefyodov <rnefyodov@yandex-team.ru> | 2022-02-10 16:47:17 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:17 +0300 |
| commit | c22320e8c4f3d7be38c504706f137034e91d31e6 (patch) | |
| tree | 35cc6c382a1bd8cb932449ffa734056fcc73e6d4 | |
| parent | 93e9e4639b6ee2afbdf45cf3927cea6d340e19b0 (diff) | |
| download | ydb-c22320e8c4f3d7be38c504706f137034e91d31e6.tar.gz | |
Restoring authorship annotation for <rnefyodov@yandex-team.ru>. Commit 1 of 2.
50 files changed, 1103 insertions, 1103 deletions
diff --git a/build/platform/clang/ya.make b/build/platform/clang/ya.make index 342b928637..57538a6644 100644 --- a/build/platform/clang/ya.make +++ b/build/platform/clang/ya.make @@ -1,9 +1,9 @@ RESOURCES_LIBRARY() - + LICENSE(BSD-3-Clause) OWNER(g:contrib) - + IF (CLANG10BC AND CLANG10 AND HOST_OS_LINUX) DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE( CLANG # Clang 10 @@ -20,5 +20,5 @@ ELSE() sbr:1006151596 FOR WIN32 # Generated by ./clang-windows.sh 7.1.0 ) ENDIF() - -END() + +END() diff --git a/build/plugins/_common.py b/build/plugins/_common.py index 2f831a94db..e6ec69b909 100644 --- a/build/plugins/_common.py +++ b/build/plugins/_common.py @@ -1,73 +1,73 @@ import sys -import hashlib +import hashlib import base64 - - -class Result(object): - pass - - -def lazy(func): - result = Result() - - def wrapper(): - try: - return result._result - except AttributeError: - result._result = func() - - return result._result - - return wrapper - - -def pathid(path): + + +class Result(object): + pass + + +def lazy(func): + result = Result() + + def wrapper(): + try: + return result._result + except AttributeError: + result._result = func() + + return result._result + + return wrapper + + +def pathid(path): return base64.b32encode(hashlib.md5(path).digest()).lower().strip('=') - - -def listid(l): - return pathid(str(sorted(l))) - - -def unpair(lst): - for x, y in lst: - yield x - yield y - - -def iterpair(lst): - y = None - - for x in lst: - if y: - yield (y, x) - - y = None - else: - y = x - - -def stripext(fname): - return fname[:fname.rfind('.')] - - -def tobuilddir(fname): + + +def listid(l): + return pathid(str(sorted(l))) + + +def unpair(lst): + for x, y in lst: + yield x + yield y + + +def iterpair(lst): + y = None + + for x in lst: + if y: + yield (y, x) + + y = None + else: + y = x + + +def stripext(fname): + return fname[:fname.rfind('.')] + + +def tobuilddir(fname): if not fname: return '$B' - if fname.startswith('$S'): - return fname.replace('$S', '$B', 1) - else: - return fname - - -def before(s, ss): - p = s.find(ss) - - if p == -1: - return s - - return s[:p] - + if fname.startswith('$S'): + return fname.replace('$S', '$B', 1) + else: + return fname + + +def before(s, ss): + p = s.find(ss) + + if p == -1: + return s + + return s[:p] + def sort_by_keywords(keywords, args): flat = [] @@ -139,32 +139,32 @@ def resolve_includes(unit, src, paths): return unit.resolve_include([src] + paths) if paths else [] -def rootrel_arc_src(src, unit): - if src.startswith('${ARCADIA_ROOT}/'): - return src[16:] - +def rootrel_arc_src(src, unit): + if src.startswith('${ARCADIA_ROOT}/'): + return src[16:] + if src.startswith('${ARCADIA_BUILD_ROOT}/'): return src[22:] - elif src.startswith('${CURDIR}/'): - return unit.path()[3:] + '/' + src[10:] - - else: - resolved = unit.resolve_arc_path(src) - - if resolved.startswith('$S/'): - return resolved[3:] - - return src # leave as is - - -def skip_build_root(x): - if x.startswith('${ARCADIA_BUILD_ROOT}'): - return x[len('${ARCADIA_BUILD_ROOT}'):].lstrip('/') - - return x - - + elif src.startswith('${CURDIR}/'): + return unit.path()[3:] + '/' + src[10:] + + else: + resolved = unit.resolve_arc_path(src) + + if resolved.startswith('$S/'): + return resolved[3:] + + return src # leave as is + + +def skip_build_root(x): + if x.startswith('${ARCADIA_BUILD_ROOT}'): + return x[len('${ARCADIA_BUILD_ROOT}'):].lstrip('/') + + return x + + def get_interpreter_path(): interpreter_path = [sys.executable] if 'ymake' in interpreter_path[0]: diff --git a/build/plugins/_import_wrapper.py b/build/plugins/_import_wrapper.py index 883f662314..5b6e33af3f 100644 --- a/build/plugins/_import_wrapper.py +++ b/build/plugins/_import_wrapper.py @@ -12,9 +12,9 @@ try: return subst(path) except ImportError: - from _custom_command import CustomCommand # noqa - from _custom_command import addrule # noqa - from _custom_command import addparser # noqa + from _custom_command import CustomCommand # noqa + from _custom_command import addrule # noqa + from _custom_command import addparser # noqa try: diff --git a/build/plugins/_unpickler.py b/build/plugins/_unpickler.py index e01e7b3118..269ab8de22 100644 --- a/build/plugins/_unpickler.py +++ b/build/plugins/_unpickler.py @@ -1,7 +1,7 @@ -import sys - -sys.dont_write_bytecode = True - +import sys + +sys.dont_write_bytecode = True + import argparse import base64 try: diff --git a/build/plugins/llvm_bc.py b/build/plugins/llvm_bc.py index 2cfe43884c..2fcac191ae 100644 --- a/build/plugins/llvm_bc.py +++ b/build/plugins/llvm_bc.py @@ -1,19 +1,19 @@ -import sys - -from _common import rootrel_arc_src, sort_by_keywords, skip_build_root, stripext - - -def onllvm_bc(unit, *args): +import sys + +from _common import rootrel_arc_src, sort_by_keywords, skip_build_root, stripext + + +def onllvm_bc(unit, *args): free_args, kwds = sort_by_keywords({'SYMBOLS': -1, 'NAME': 1, 'NO_COMPILE': 0}, args) - name = kwds['NAME'][0] - symbols = kwds.get('SYMBOLS') + name = kwds['NAME'][0] + symbols = kwds.get('SYMBOLS') obj_suf = unit.get('OBJ_SUF') skip_compile_step = 'NO_COMPILE' in kwds merged_bc = name + '_merged' + obj_suf + '.bc' out_bc = name + '_optimized' + obj_suf + '.bc' - bcs = [] - for x in free_args: - rel_path = rootrel_arc_src(x, unit) + bcs = [] + for x in free_args: + rel_path = rootrel_arc_src(x, unit) bc_path = '${ARCADIA_BUILD_ROOT}/' + skip_build_root(rel_path) + obj_suf + '.bc' if not skip_compile_step: if x.endswith('.c'): @@ -23,11 +23,11 @@ def onllvm_bc(unit, *args): else: llvm_compile = unit.onllvm_compile_cxx llvm_compile([rel_path, bc_path]) - bcs.append(bc_path) - unit.onllvm_link([merged_bc] + bcs) - opt_opts = ['-O2', '-globalopt', '-globaldce'] - if symbols: - # XXX: '#' used instead of ',' to overcome ymake tendency to split everything by comma - opt_opts += ['-internalize', '-internalize-public-api-list=' + '#'.join(symbols)] - unit.onllvm_opt([merged_bc, out_bc] + opt_opts) - unit.onresource([out_bc, '/llvm_bc/' + name]) + bcs.append(bc_path) + unit.onllvm_link([merged_bc] + bcs) + opt_opts = ['-O2', '-globalopt', '-globaldce'] + if symbols: + # XXX: '#' used instead of ',' to overcome ymake tendency to split everything by comma + opt_opts += ['-internalize', '-internalize-public-api-list=' + '#'.join(symbols)] + unit.onllvm_opt([merged_bc, out_bc] + opt_opts) + unit.onresource([out_bc, '/llvm_bc/' + name]) diff --git a/build/plugins/pybuild.py b/build/plugins/pybuild.py index f32a2d39a0..0d722f385e 100644 --- a/build/plugins/pybuild.py +++ b/build/plugins/pybuild.py @@ -43,8 +43,8 @@ def pb2_arg(suf, path, mod, unit): modsuf=stripext(suf) ) -def proto_arg(path, mod, unit): - return '{}.proto={}'.format(stripext(to_build_root(path, unit)), mod) +def proto_arg(path, mod, unit): + return '{}.proto={}'.format(stripext(to_build_root(path, unit)), mod) def pb_cc_arg(suf, path, unit): return '{}{suf}'.format(stripext(to_build_root(path, unit)), suf=suf) @@ -311,7 +311,7 @@ def onpy_srcs(unit, *args): mod_root_path = root_rel_path[:-(len(path) + 1)] py_namespaces.setdefault(mod_root_path, set()).add(ns if ns else '.') mod = ns + mod_name - + if main_mod: py_main(unit, mod + ":main") elif py3 and unit_needs_main and main_py: diff --git a/build/plugins/res.py b/build/plugins/res.py index a937caba81..0c47e6ed7e 100644 --- a/build/plugins/res.py +++ b/build/plugins/res.py @@ -1,5 +1,5 @@ from _common import iterpair, listid, pathid, rootrel_arc_src, tobuilddir, filter_out_by_keyword - + def split(lst, limit): # paths are specified with replaceable prefix @@ -36,7 +36,7 @@ def remove_prefix(text, prefix): def onfat_resource(unit, *args): unit.onpeerdir(['library/cpp/resource']) - + # Since the maximum length of lpCommandLine string for CreateProcess is 8kb (windows) characters, # we make several calls of rescompiler # https://msdn.microsoft.com/ru-ru/library/windows/desktop/ms682425.aspx @@ -45,7 +45,7 @@ def onfat_resource(unit, *args): inputs = [x for x, y in iterpair(part_args) if x != '-'] if inputs: inputs = ['IN'] + inputs - + unit.onrun_program(['tools/rescompiler', output] + part_args + inputs + ['OUT_NOAUTO', output]) unit.onsrcs(['GLOBAL', output]) diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py index 8970837f0f..b70d767e47 100644 --- a/build/plugins/ytest.py +++ b/build/plugins/ytest.py @@ -12,7 +12,7 @@ import _requirements as reqs import StringIO import subprocess import collections - + import ymake @@ -131,7 +131,7 @@ def validate_test(unit, kw): if valid_kw.get('SCRIPT-REL-PATH') == 'boost.test': project_path = valid_kw.get('BUILD-FOLDER-PATH', "") - if not project_path.startswith(("contrib", "mail", "maps", "tools/idl", "metrika", "devtools", "mds", "yandex_io", "smart_devices")): + if not project_path.startswith(("contrib", "mail", "maps", "tools/idl", "metrika", "devtools", "mds", "yandex_io", "smart_devices")): errors.append("BOOSTTEST is not allowed here") elif valid_kw.get('SCRIPT-REL-PATH') == 'gtest': project_path = valid_kw.get('BUILD-FOLDER-PATH', "") diff --git a/build/scripts/clang_wrapper.py b/build/scripts/clang_wrapper.py index af3869f789..fb34497567 100644 --- a/build/scripts/clang_wrapper.py +++ b/build/scripts/clang_wrapper.py @@ -3,12 +3,12 @@ import sys def fix(s): - # disable dbg DEVTOOLS-2744 - if s == '-g': - return None + # disable dbg DEVTOOLS-2744 + if s == '-g': + return None if s == '/Z7' or s == '/Od' or s == '/Ob0' or s == '/D_DEBUG': return None - + # disable sanitizers for generated code if s.startswith('-fsanitize') or s == '-Dmemory_sanitizer_enabled' or s.startswith('-fsanitize-blacklist'): return None diff --git a/build/scripts/coverage-info.py b/build/scripts/coverage-info.py index 94491d9256..cca68ac27f 100644 --- a/build/scripts/coverage-info.py +++ b/build/scripts/coverage-info.py @@ -1,17 +1,17 @@ -import argparse -import os -import sys -import tarfile -import collections -import subprocess +import argparse +import os +import sys +import tarfile +import collections +import subprocess import re - - -GCDA_EXT = '.gcda' -GCNO_EXT = '.gcno' - - -def suffixes(path): + + +GCDA_EXT = '.gcda' +GCNO_EXT = '.gcno' + + +def suffixes(path): """ >>> list(suffixes('/a/b/c')) ['c', 'b/c', '/a/b/c'] @@ -24,67 +24,67 @@ def suffixes(path): >>> list(suffixes('/')) [] """ - path = os.path.normpath(path) - - def up_dirs(cur_path): - while os.path.dirname(cur_path) != cur_path: - cur_path = os.path.dirname(cur_path) - yield cur_path - - for x in up_dirs(path): - yield path.replace(x + os.path.sep, '') - - -def recast(in_file, out_file, probe_path, update_stat): - PREFIX = 'SF:' - - probed_path = None - - any_payload = False - - with open(in_file, 'r') as input, open(out_file, 'w') as output: - active = True - for line in input: - line = line.rstrip('\n') - if line.startswith('TN:'): - output.write(line + '\n') - elif line.startswith(PREFIX): - path = line[len(PREFIX):] - probed_path = probe_path(path) - if probed_path: - output.write(PREFIX + probed_path + '\n') - active = bool(probed_path) - else: - if active: - update_stat(probed_path, line) - output.write(line + '\n') - any_payload = True - - return any_payload - - -def print_stat(da, fnda, teamcity_stat_output): - lines_hit = sum(map(bool, da.values())) - lines_total = len(da.values()) - lines_coverage = 100.0 * lines_hit / lines_total if lines_total else 0 - - func_hit = sum(map(bool, fnda.values())) - func_total = len(fnda.values()) - func_coverage = 100.0 * func_hit / func_total if func_total else 0 - - print >>sys.stderr, '[[imp]]Lines[[rst]] {: >16} {: >16} {: >16.1f}%'.format(lines_hit, lines_total, lines_coverage) - print >>sys.stderr, '[[imp]]Functions[[rst]] {: >16} {: >16} {: >16.1f}%'.format(func_hit, func_total, func_coverage) - - if teamcity_stat_output: - with open(teamcity_stat_output, 'w') as tc_file: - tc_file.write("##teamcity[blockOpened name='Code Coverage Summary']\n") - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLTotal\' value='{}']\n".format(lines_total)) - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLCovered\' value='{}']\n".format(lines_hit)) - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMTotal\' value='{}']\n".format(func_total)) - tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMCovered\' value='{}']\n".format(func_hit)) - tc_file.write("##teamcity[blockClosed name='Code Coverage Summary']\n") - - + path = os.path.normpath(path) + + def up_dirs(cur_path): + while os.path.dirname(cur_path) != cur_path: + cur_path = os.path.dirname(cur_path) + yield cur_path + + for x in up_dirs(path): + yield path.replace(x + os.path.sep, '') + + +def recast(in_file, out_file, probe_path, update_stat): + PREFIX = 'SF:' + + probed_path = None + + any_payload = False + + with open(in_file, 'r') as input, open(out_file, 'w') as output: + active = True + for line in input: + line = line.rstrip('\n') + if line.startswith('TN:'): + output.write(line + '\n') + elif line.startswith(PREFIX): + path = line[len(PREFIX):] + probed_path = probe_path(path) + if probed_path: + output.write(PREFIX + probed_path + '\n') + active = bool(probed_path) + else: + if active: + update_stat(probed_path, line) + output.write(line + '\n') + any_payload = True + + return any_payload + + +def print_stat(da, fnda, teamcity_stat_output): + lines_hit = sum(map(bool, da.values())) + lines_total = len(da.values()) + lines_coverage = 100.0 * lines_hit / lines_total if lines_total else 0 + + func_hit = sum(map(bool, fnda.values())) + func_total = len(fnda.values()) + func_coverage = 100.0 * func_hit / func_total if func_total else 0 + + print >>sys.stderr, '[[imp]]Lines[[rst]] {: >16} {: >16} {: >16.1f}%'.format(lines_hit, lines_total, lines_coverage) + print >>sys.stderr, '[[imp]]Functions[[rst]] {: >16} {: >16} {: >16.1f}%'.format(func_hit, func_total, func_coverage) + + if teamcity_stat_output: + with open(teamcity_stat_output, 'w') as tc_file: + tc_file.write("##teamcity[blockOpened name='Code Coverage Summary']\n") + tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLTotal\' value='{}']\n".format(lines_total)) + tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsLCovered\' value='{}']\n".format(lines_hit)) + tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMTotal\' value='{}']\n".format(func_total)) + tc_file.write("##teamcity[buildStatisticValue key=\'CodeCoverageAbsMCovered\' value='{}']\n".format(func_hit)) + tc_file.write("##teamcity[blockClosed name='Code Coverage Summary']\n") + + def chunks(l, n): """ >>> list(chunks(range(10), 3)) @@ -118,17 +118,17 @@ def combine_info_files(lcov, files, out_file): def probe_path_global(path, source_root, prefix_filter, exclude_files): if path.endswith('_ut.cpp'): - return None - + return None + for suff in reversed(list(suffixes(path))): if (not prefix_filter or suff.startswith(prefix_filter)) and (not exclude_files or not exclude_files.match(suff)): full_path = source_root + os.sep + suff if os.path.isfile(full_path): return full_path - + return None - - + + def update_stat_global(src_file, line, fnda, da): if line.startswith("FNDA:"): visits, func_name = line[len("FNDA:"):].split(',') @@ -179,27 +179,27 @@ def init_all_coverage_files(gcno_archive, fname2gcno, fname2info, geninfo_execut def process_all_coverage_files(gcda_archive, fname2gcno, fname2info, geninfo_executable, gcov_tool, gen_info): - with tarfile.open(gcda_archive) as gcda_tf: - for gcda_item in gcda_tf: - if gcda_item.isfile() and gcda_item.name.endswith(GCDA_EXT): - gcda_name = gcda_item.name - source_fname = gcda_name[:-len(GCDA_EXT)] - for suff in suffixes(source_fname): - if suff in fname2gcno: + with tarfile.open(gcda_archive) as gcda_tf: + for gcda_item in gcda_tf: + if gcda_item.isfile() and gcda_item.name.endswith(GCDA_EXT): + gcda_name = gcda_item.name + source_fname = gcda_name[:-len(GCDA_EXT)] + for suff in suffixes(source_fname): + if suff in fname2gcno: gcda_new_name = suff + GCDA_EXT - gcda_item.name = gcda_new_name - gcda_tf.extract(gcda_item) - if os.path.getsize(gcda_new_name) > 0: + gcda_item.name = gcda_new_name + gcda_tf.extract(gcda_item) + if os.path.getsize(gcda_new_name) > 0: coverage_info = suff + '.' + str(len(fname2info[suff])) + '.info' fname2info[suff].append(coverage_info) - geninfo_cmd = [ + geninfo_cmd = [ geninfo_executable, - '--gcov-tool', gcov_tool, - gcda_new_name, + '--gcov-tool', gcov_tool, + gcda_new_name, '-o', coverage_info + '.tmp' - ] + ] gen_info(geninfo_cmd, coverage_info) - + def gen_cobertura(tool, output, combined_info): cmd = [ @@ -241,42 +241,42 @@ def main(source_root, output, gcno_archive, gcda_archive, gcov_tool, prefix_filt output_dir = coverage_report_path else: output_dir = output + '.dir' - + if not os.path.exists(output_dir): os.makedirs(output_dir) - teamcity_stat_file = None - if teamcity_stat_output: - teamcity_stat_file = os.path.join(output_dir, 'teamcity.out') - print_stat(da, fnda, teamcity_stat_file) - - if lcov_args: + teamcity_stat_file = None + if teamcity_stat_output: + teamcity_stat_file = os.path.join(output_dir, 'teamcity.out') + print_stat(da, fnda, teamcity_stat_file) + + if lcov_args: output_trace = "combined.info" combine_info_files(os.path.join(source_root, 'devtools', 'lcov', 'lcov'), lcov_args, output_trace) cmd = [os.path.join(source_root, 'devtools', 'lcov', 'genhtml'), '-p', source_root, '--ignore-errors', 'source', '-o', output_dir, output_trace] - print >>sys.stderr, '## genhtml', ' '.join(cmd) - subprocess.check_call(cmd) + print >>sys.stderr, '## genhtml', ' '.join(cmd) + subprocess.check_call(cmd) if lcov_cobertura: gen_cobertura(lcov_cobertura, gcov_report, output_trace) - - with tarfile.open(output, 'w') as tar: - tar.add(output_dir, arcname='.') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('--source-root', action='store') - parser.add_argument('--output', action='store') - parser.add_argument('--gcno-archive', action='store') - parser.add_argument('--gcda-archive', action='store') - parser.add_argument('--gcov-tool', action='store') - parser.add_argument('--prefix-filter', action='store') + + with tarfile.open(output, 'w') as tar: + tar.add(output_dir, arcname='.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument('--source-root', action='store') + parser.add_argument('--output', action='store') + parser.add_argument('--gcno-archive', action='store') + parser.add_argument('--gcda-archive', action='store') + parser.add_argument('--gcov-tool', action='store') + parser.add_argument('--prefix-filter', action='store') parser.add_argument('--exclude-regexp', action='store') - parser.add_argument('--teamcity-stat-output', action='store_const', const=True) + parser.add_argument('--teamcity-stat-output', action='store_const', const=True) parser.add_argument('--coverage-report-path', action='store') parser.add_argument('--gcov-report', action='store') parser.add_argument('--lcov-cobertura', action='store') - - args = parser.parse_args() - main(**vars(args)) + + args = parser.parse_args() + main(**vars(args)) diff --git a/build/scripts/error.py b/build/scripts/error.py index f7d8ecb2cc..644a172d89 100644 --- a/build/scripts/error.py +++ b/build/scripts/error.py @@ -8,11 +8,11 @@ TEMPORARY_ERROR_MESSAGES = [ 'Internal Server Error', 'Network connection closed unexpectedly', 'Network is unreachable', - 'No route to host', + 'No route to host', 'No space left on device', 'Not enough space', 'Temporary failure in name resolution', - 'The read operation timed out', + 'The read operation timed out', 'timeout: timed out', ] @@ -30,48 +30,48 @@ def merge_exit_codes(exit_codes): return max(e if e >= 0 else 1 for e in exit_codes) if exit_codes else 0 -def is_temporary_error(exc): +def is_temporary_error(exc): import logging logger = logging.getLogger(__name__) - if getattr(exc, 'temporary', False): + if getattr(exc, 'temporary', False): logger.debug("Exception has temporary attribute: %s", exc) - return True - - import errno - err = getattr(exc, 'errno', None) - - if err == errno.ECONNREFUSED or err == errno.ENETUNREACH: + return True + + import errno + err = getattr(exc, 'errno', None) + + if err == errno.ECONNREFUSED or err == errno.ENETUNREACH: logger.debug("Exception has errno attribute: %s (errno=%s)", exc, err) - return True - - import socket - + return True + + import socket + if isinstance(exc, socket.timeout) or isinstance(getattr(exc, 'reason', None), socket.timeout): logger.debug("Socket timeout exception: %s", exc) - return True - - if isinstance(exc, socket.gaierror): + return True + + if isinstance(exc, socket.gaierror): logger.debug("Getaddrinfo exception: %s", exc) - return True - + return True + import urllib2 if isinstance(exc, urllib2.HTTPError) and exc.code in (429, ): logger.debug("urllib2.HTTPError: %s", exc) return True - import httplib - - if isinstance(exc, httplib.IncompleteRead): + import httplib + + if isinstance(exc, httplib.IncompleteRead): logger.debug("IncompleteRead exception: %s", exc) - return True - - exc_str = str(exc) - + return True + + exc_str = str(exc) + for message in TEMPORARY_ERROR_MESSAGES: if message in exc_str: logger.debug("Found temporary error pattern (%s): %s", message, exc_str) return True - - return False + + return False diff --git a/build/scripts/fetch_from_sandbox.py b/build/scripts/fetch_from_sandbox.py index a99542e174..4203aa7a3d 100755 --- a/build/scripts/fetch_from_sandbox.py +++ b/build/scripts/fetch_from_sandbox.py @@ -3,9 +3,9 @@ import json import logging import argparse import os -import random +import random import subprocess -import sys +import sys import time import urllib2 import uuid @@ -13,11 +13,11 @@ import uuid import fetch_from -ORIGIN_SUFFIX = '?origin=fetch-from-sandbox' -MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' +ORIGIN_SUFFIX = '?origin=fetch-from-sandbox' +MDS_PREFIX = 'http://storage-int.mds.yandex.net/get-sandbox/' TEMPORARY_ERROR_CODES = (429, 500, 503, 504) - - + + def parse_args(): parser = argparse.ArgumentParser() fetch_from.add_common_arguments(parser) @@ -128,10 +128,10 @@ def get_resource_http_links(resource_id): return [r['url'] + ORIGIN_SUFFIX for r in _query(url)] -def fetch_via_script(script, resource_id): - return subprocess.check_output([script, str(resource_id)]).rstrip() - - +def fetch_via_script(script, resource_id): + return subprocess.check_output([script, str(resource_id)]).rstrip() + + def fetch(resource_id, custom_fetcher): try: resource_info = get_resource_info(resource_id, touch=True, no_links=True) @@ -146,14 +146,14 @@ def fetch(resource_id, custom_fetcher): logging.info('Resource %s info %s', str(resource_id), json.dumps(resource_info)) - resource_file_name = os.path.basename(resource_info["file_name"]) - expected_md5 = resource_info.get('md5') - - proxy_link = resource_info['http']['proxy'] + ORIGIN_SUFFIX - - mds_id = resource_info.get('attributes', {}).get('mds') - mds_link = MDS_PREFIX + mds_id if mds_id else None - + resource_file_name = os.path.basename(resource_info["file_name"]) + expected_md5 = resource_info.get('md5') + + proxy_link = resource_info['http']['proxy'] + ORIGIN_SUFFIX + + mds_id = resource_info.get('attributes', {}).get('mds') + mds_link = MDS_PREFIX + mds_id if mds_id else None + def get_storage_links(): storage_links = get_resource_http_links(resource_id) random.shuffle(storage_links) @@ -164,33 +164,33 @@ def fetch(resource_id, custom_fetcher): if not skynet: logging.info("Skynet is not available, will try other protocols") - def iter_tries(): + def iter_tries(): if skynet: yield lambda: download_by_skynet(resource_info, resource_file_name) - if custom_fetcher: - yield lambda: fetch_via_script(custom_fetcher, resource_id) + if custom_fetcher: + yield lambda: fetch_via_script(custom_fetcher, resource_id) # Don't try too hard here: we will get back to proxy later on yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5, tries=2) for x in get_storage_links(): # Don't spend too much time connecting single host yield lambda: fetch_from.fetch_url(x, False, resource_file_name, expected_md5, tries=1) - if mds_link is not None: + if mds_link is not None: # Don't try too hard here: we will get back to MDS later on yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5, tries=2) yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5) - if mds_link is not None: + if mds_link is not None: yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5) - + if resource_info.get('attributes', {}).get('ttl') != 'inf': sys.stderr.write('WARNING: resource {} ttl is not "inf".\n'.format(resource_id)) - exc_info = None - for i, action in enumerate(itertools.islice(iter_tries(), 0, 10)): - try: - fetched_file = action() - break + exc_info = None + for i, action in enumerate(itertools.islice(iter_tries(), 0, 10)): + try: + fetched_file = action() + break except UnsupportedProtocolException: pass except subprocess.CalledProcessError as e: @@ -201,18 +201,18 @@ def fetch(resource_id, custom_fetcher): if e.code not in TEMPORARY_ERROR_CODES: exc_info = exc_info or sys.exc_info() time.sleep(i) - except Exception as e: - logging.exception(e) - exc_info = exc_info or sys.exc_info() - time.sleep(i) - else: + except Exception as e: + logging.exception(e) + exc_info = exc_info or sys.exc_info() + time.sleep(i) + else: if exc_info: raise exc_info[0], exc_info[1], exc_info[2] else: raise Exception("No available protocol and/or server to fetch resource") return fetched_file, resource_info['file_name'] - + def _get_resource_info_from_file(resource_file): if resource_file is None or not os.path.exists(resource_file): @@ -254,16 +254,16 @@ def main(args): fetch_from.process(fetched_file, file_name, args, remove=not custom_fetcher and not resource_info) -if __name__ == '__main__': +if __name__ == '__main__': args = parse_args() fetch_from.setup_logging(args, os.path.basename(__file__)) - try: + try: main(args) - except Exception as e: - logging.exception(e) + except Exception as e: + logging.exception(e) print >>sys.stderr, open(args.abs_log_path).read() - sys.stderr.flush() + sys.stderr.flush() import error sys.exit(error.ExitCodes.INFRASTRUCTURE_ERROR if fetch_from.is_temporary(e) else 1) diff --git a/build/scripts/find_and_tar.py b/build/scripts/find_and_tar.py index f251623c68..1fe17fc743 100644 --- a/build/scripts/find_and_tar.py +++ b/build/scripts/find_and_tar.py @@ -1,22 +1,22 @@ -import os -import sys -import tarfile - - -def find_gcno(dirname, tail): - for cur, _dirs, files in os.walk(dirname): - for f in files: - if f.endswith(tail): - yield os.path.relpath(os.path.join(cur, f)) - - -def main(args): - output = args[0] +import os +import sys +import tarfile + + +def find_gcno(dirname, tail): + for cur, _dirs, files in os.walk(dirname): + for f in files: + if f.endswith(tail): + yield os.path.relpath(os.path.join(cur, f)) + + +def main(args): + output = args[0] tail = args[1] if len(args) > 1 else '' - with tarfile.open(output, 'w:') as tf: - for f in find_gcno(os.getcwd(), tail): - tf.add(f) - - -if __name__ == '__main__': - main(sys.argv[1:]) + with tarfile.open(output, 'w:') as tf: + for f in find_gcno(os.getcwd(), tail): + tf.add(f) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/scripts/gen_tasklet_reg.py b/build/scripts/gen_tasklet_reg.py index 5b747c2eca..ed2f1a89ba 100644 --- a/build/scripts/gen_tasklet_reg.py +++ b/build/scripts/gen_tasklet_reg.py @@ -3,9 +3,9 @@ import argparse TEMPLATE = '''\ {includes}\ #include <tasklet/runtime/lib/{language}_wrapper.h> -#include <tasklet/runtime/lib/registry.h> +#include <tasklet/runtime/lib/registry.h> -static const NTasklet::TRegHelper REG( +static const NTasklet::TRegHelper REG( "{name}", new NTasklet::{wrapper} ); diff --git a/build/scripts/generate_mf.py b/build/scripts/generate_mf.py index a44a969980..e1cdc18784 100644 --- a/build/scripts/generate_mf.py +++ b/build/scripts/generate_mf.py @@ -106,8 +106,8 @@ def generate_mf(): if __name__ == '__main__': - try: - generate_mf() - except Exception as e: - sys.stderr.write(str(e) + '\n') - sys.exit(1) + try: + generate_mf() + except Exception as e: + sys.stderr.write(str(e) + '\n') + sys.exit(1) diff --git a/build/scripts/llvm_opt_wrapper.py b/build/scripts/llvm_opt_wrapper.py index 38ca3004af..d3a14b3738 100644 --- a/build/scripts/llvm_opt_wrapper.py +++ b/build/scripts/llvm_opt_wrapper.py @@ -1,18 +1,18 @@ import subprocess -import sys - - -def fix(s): - # we use '#' instead of ',' because ymake always splits args by comma - if 'internalize' in s: - return s.replace('#', ',') - - return s - - -if __name__ == '__main__': - path = sys.argv[1] - args = [fix(s) for s in [path] + sys.argv[2:]] - +import sys + + +def fix(s): + # we use '#' instead of ',' because ymake always splits args by comma + if 'internalize' in s: + return s.replace('#', ',') + + return s + + +if __name__ == '__main__': + path = sys.argv[1] + args = [fix(s) for s in [path] + sys.argv[2:]] + rc = subprocess.call(args, shell=False, stderr=sys.stderr, stdout=sys.stdout) sys.exit(rc) diff --git a/build/scripts/merge_coverage_data.py b/build/scripts/merge_coverage_data.py index b7fa3c6a86..75ed820033 100644 --- a/build/scripts/merge_coverage_data.py +++ b/build/scripts/merge_coverage_data.py @@ -1,11 +1,11 @@ -import sys -import tarfile +import sys +import tarfile import copy import os import uuid - - -def main(args): + + +def main(args): output_file, args = args[0], args[1:] # heretic@: Splits files on which could be merged( files ) and which should not be merged( expendables ) # expendables will be in output_file in form {name}{ordinal number of archive in args[]}.{extension} @@ -14,8 +14,8 @@ def main(args): except ValueError: split_i = len(args) files, expendables = args[:split_i], args[split_i + 1:] - - with tarfile.open(output_file, 'w') as outf: + + with tarfile.open(output_file, 'w') as outf: for x in files: with tarfile.open(x) as tf: for tarinfo in tf: @@ -26,7 +26,7 @@ def main(args): new_basename = '.'.join([basename_parts[0] + str(uuid.uuid4())] + basename_parts[1:]) new_tarinfo.name = os.path.join(dirname, new_basename) outf.addfile(new_tarinfo, tf.extractfile(tarinfo)) - - -if __name__ == '__main__': - main(sys.argv[1:]) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/scripts/perl_wrapper.py b/build/scripts/perl_wrapper.py index cb4027f1d3..2a0a20b6c7 100644 --- a/build/scripts/perl_wrapper.py +++ b/build/scripts/perl_wrapper.py @@ -1,7 +1,7 @@ -import os -import sys +import os +import sys import shutil - + if __name__ == '__main__': path = sys.argv[1] to = sys.argv[-1] diff --git a/build/scripts/with_coverage.py b/build/scripts/with_coverage.py index d62435c3b8..29216d96a7 100644 --- a/build/scripts/with_coverage.py +++ b/build/scripts/with_coverage.py @@ -1,40 +1,40 @@ # TODO prettyboy remove after ya-bin release -import os -import sys -import subprocess -import tarfile -import random -import shutil - - -def mkdir_p(path): - try: - os.makedirs(path) - except OSError: - pass - - -def main(args): - coverage_path = os.path.abspath(args[0]) - coverage_dir = coverage_path + '.' + str(random.getrandbits(64)) - - mkdir_p(coverage_dir) - - env = os.environ.copy() - env['GCOV_PREFIX'] = coverage_dir - - subprocess.check_call(args[1:], env=env) - - arch_path = coverage_dir + '.archive' - - with tarfile.open(arch_path, 'w:') as tar: - tar.add(coverage_dir, arcname='.') - - os.rename(arch_path, coverage_path) - - shutil.rmtree(coverage_dir) - - -if __name__ == '__main__': - main(sys.argv[1:]) +import os +import sys +import subprocess +import tarfile +import random +import shutil + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError: + pass + + +def main(args): + coverage_path = os.path.abspath(args[0]) + coverage_dir = coverage_path + '.' + str(random.getrandbits(64)) + + mkdir_p(coverage_dir) + + env = os.environ.copy() + env['GCOV_PREFIX'] = coverage_dir + + subprocess.check_call(args[1:], env=env) + + arch_path = coverage_dir + '.archive' + + with tarfile.open(arch_path, 'w:') as tar: + tar.add(coverage_dir, arcname='.') + + os.rename(arch_path, coverage_path) + + shutil.rmtree(coverage_dir) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/build/scripts/ya.make b/build/scripts/ya.make index 710165e40d..f74d16b11f 100644 --- a/build/scripts/ya.make +++ b/build/scripts/ya.make @@ -1,5 +1,5 @@ OWNER(g:ymake) - + PY2TEST() TEST_SRCS( @@ -57,7 +57,7 @@ TEST_SRCS( link_exe.py link_fat_obj.py link_lib.py - llvm_opt_wrapper.py + llvm_opt_wrapper.py merge_coverage_data.py merge_files.py mkdir.py @@ -92,11 +92,11 @@ TEST_SRCS( writer.py xargs.py yield_line.py - yndexer.py + yndexer.py ) PEERDIR( ydb/library/yql/public/udf ) -END() +END() diff --git a/build/scripts/yndexer.py b/build/scripts/yndexer.py index a38e28ba99..71a6167ae4 100644 --- a/build/scripts/yndexer.py +++ b/build/scripts/yndexer.py @@ -1,9 +1,9 @@ -import sys -import subprocess +import sys +import subprocess import threading -import os +import os import re - + rx_resource_dir = re.compile(r'libraries: =([^:]*)') @@ -41,19 +41,19 @@ class Process(object): return self._result -if __name__ == '__main__': - args = sys.argv - - yndexer = args[1] +if __name__ == '__main__': + args = sys.argv + + yndexer = args[1] timeout = int(args[2]) arc_root = args[3] build_root = args[4] input_file = args[5] output_file = args[-1] tail_args = args[6:-1] - - subprocess.check_call(tail_args) - + + subprocess.check_call(tail_args) + clang = tail_args[0] out = subprocess.check_output([clang, '-print-search-dirs']) resource_dir = rx_resource_dir.search(out).group(1) @@ -70,7 +70,7 @@ if __name__ == '__main__': ] + tail_args + [ '-resource-dir', resource_dir, ] - + process = Process(yndexer_args) result = process.wait(timeout=timeout) diff --git a/build/sysincl/unsorted.yml b/build/sysincl/unsorted.yml index a889b923a0..a1cc24cbd5 100644 --- a/build/sysincl/unsorted.yml +++ b/build/sysincl/unsorted.yml @@ -493,7 +493,7 @@ - altivec.h #endif #if defined(__vxWorks__) - - vxCpuLib.h + - vxCpuLib.h - vxWorks.h - sockLib.h - hostLib.h diff --git a/build/ya.conf.json b/build/ya.conf.json index 5f7cc875d6..ff9a204f9f 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -1,5 +1,5 @@ { - "tools": { + "tools": { "cc": { "description": "Run C compiler" }, @@ -511,8 +511,8 @@ "exp_stats": { "description": "bsyeti tool" } - }, - "toolchain": { + }, + "toolchain": { "bigb_request": { "tools": { "bigb_request": { @@ -1025,13 +1025,13 @@ } ] }, - "ymake": { - "tools": { + "ymake": { + "tools": { "ymake": { "bottle": "ymake", "executable": "ymake" } - }, + }, "platforms": [ { "host": { @@ -1073,7 +1073,7 @@ "default": true } ] - }, + }, "maven_import_sandbox_uploader": { "tools": { "maven_import_sandbox_uploader": { @@ -2706,13 +2706,13 @@ } ] }, - "gdb": { - "tools": { + "gdb": { + "tools": { "gdb": { "bottle": "gdb", "executable": "gdb" } - }, + }, "platforms": [ { "host": { @@ -2749,10 +2749,10 @@ ], "env": { "TERMINFO": [ - "$(ROOT)/gdb/lib/terminfo" + "$(ROOT)/gdb/lib/terminfo" ] } - }, + }, "gdbserver": { "tools": { "gdbserver": { @@ -3492,13 +3492,13 @@ } ] }, - "valgrind": { - "tools": { + "valgrind": { + "tools": { "valgrind": { "bottle": "valgrind", "executable": "valgrind" } - }, + }, "platforms": [ { "host": { @@ -3650,13 +3650,13 @@ } ] }, - "jq": { - "tools": { + "jq": { + "tools": { "jq": { "bottle": "jq", "executable": "jq" } - }, + }, "platforms": [ { "host": { @@ -3665,7 +3665,7 @@ "default": true } ] - }, + }, "vim": { "tools": { "vim": { @@ -3769,8 +3769,8 @@ } ] }, - "kiwi": { - "tools": { + "kiwi": { + "tools": { "kwmktorrent": { "bottle": "kwmktorrent", "executable": "kwmktorrent" @@ -3799,7 +3799,7 @@ "bottle": "kwworm", "executable": "kwworm" } - }, + }, "platforms": [ { "host": { @@ -3808,7 +3808,7 @@ "default": true } ] - }, + }, "optrace": { "tools": { "optrace": { @@ -3885,14 +3885,14 @@ "default": true } ] - }, - "ant": { - "tools": { + }, + "ant": { + "tools": { "ant": { "bottle": "ant", "executable": "ant" } - }, + }, "platforms": [ { "host": { @@ -4349,7 +4349,7 @@ "default": true } ] - }, + }, "rtags": { "tools": { "rtags": { @@ -4371,14 +4371,14 @@ } ] }, - "afl-fuzz": { - "tools": { + "afl-fuzz": { + "tools": { "afl-fuzz": { "bottle": "afl-fuzz", "executable": "afl-fuzz" } - }, - "platforms": [ + }, + "platforms": [ { "host": { "os": "LINUX" @@ -4391,7 +4391,7 @@ }, "default": true } - ] + ] }, "horadric": { "tools": { @@ -5721,9 +5721,9 @@ "default": true } ] - } - }, - "bottles": { + } + }, + "bottles": { "msvc2019": { "formula": { "sandbox_id": 887495315, @@ -5738,19 +5738,19 @@ ] } }, - "ymake": { - "formula": { + "ymake": { + "formula": { "sandbox_id": [ 1206141440 ], "match": "ymake" - }, - "executable": { + }, + "executable": { "ymake": [ "ymake" ] - } - }, + } + }, "clang-format": { "formula": { "sandbox_id": [ @@ -6722,17 +6722,17 @@ ] } }, - "gdb": { - "formula": { + "gdb": { + "formula": { "sandbox_id": [ 1037272933, 1032891985, 717207102, 1032896450 ], - "match": "GDB" - }, - "executable": { + "match": "GDB" + }, + "executable": { "gdb": [ "gdb", "bin", @@ -6743,8 +6743,8 @@ "bin", "gdbserver" ] - } - }, + } + }, "pprof": { "formula": { "sandbox_id": 1115456319, @@ -6779,32 +6779,32 @@ ] } }, - "valgrind": { - "formula": { + "valgrind": { + "formula": { "sandbox_id": 944969271, - "match": "Valgrind" - }, - "executable": { + "match": "Valgrind" + }, + "executable": { "valgrind": [ "valgrind" ] - } - }, - "jq": { - "formula": { + } + }, + "jq": { + "formula": { "sandbox_id": [ 38336932, 38337150, 38337983 ], - "match": "jq" - }, - "executable": { + "match": "jq" + }, + "executable": { "jq": [ "jq" ] - } - }, + } + }, "yf": { "formula": { "sandbox_id": [ @@ -7092,25 +7092,25 @@ "kwmqbuild" ] } - }, + }, "gpt": { "formula": { "sandbox_id": 21836503, "match": "GooglePerformanceTools" } - }, - "ant": { - "formula": { + }, + "ant": { + "formula": { "sandbox_id": 36996223, - "match": "ant" - }, - "executable": { + "match": "ant" + }, + "executable": { "ant": [ "apache-ant-1.9.6", "bin", "ant" ] - } + } }, "maven": { "formula": { @@ -7173,16 +7173,16 @@ } }, "afl-fuzz": { - "formula": { - "sandbox_id": 80600007, - "match": "afl-fuzz" - }, - "executable": { + "formula": { + "sandbox_id": 80600007, + "match": "afl-fuzz" + }, + "executable": { "afl-fuzz": [ "afl-fuzz" ] - } - }, + } + }, "horadric": { "formula": { "sandbox_id": 1210601101, @@ -8089,11 +8089,11 @@ "profile_size_analyzer" ] } - } + } }, "toolchain_aliases": { "clang-win-x86_64": "clang12-win-x86_64", "clang-win-i686": "clang12-win-i686", "gcc-linux-x86_64": "gcc82-linux-x86_64" - } + } } diff --git a/build/ya.make b/build/ya.make index 407b8f13e0..1fafe2f67d 100644 --- a/build/ya.make +++ b/build/ya.make @@ -6,7 +6,7 @@ OWNER( NEED_CHECK() PY2_LIBRARY() - + PY_SRCS( ymake_conf.py ) @@ -17,8 +17,8 @@ PEERDIR( library/cpp/string_utils/scan ) -END() - +END() + RECURSE( conf_fatal_error config @@ -32,7 +32,7 @@ RECURSE( plugins prebuilt scripts - tests + tests tests_slow yandex_specific/config ) diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 081833998b..738854c83e 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -7,19 +7,19 @@ # - Autocheck will rebuild and recache everything. # Use this with extreme care and only change if it is utlimately needed. Consider more specific XXX_FAKEIDs below instead. FAKEID=3141592653 - + SANDBOX_FAKEID=${FAKEID}.7600000 CPP_FAKEID=9107927 GO_FAKEID=9056219 ANDROID_FAKEID=8821472 CLANG_TIDY_FAKEID=8625699 -CURDIR=. +CURDIR=. MODDIR=. -BINDIR=bin:/ -SRCDIR= -YMAKE=ymake -ECHO=echo +BINDIR=bin:/ +SRCDIR= +YMAKE=ymake +ECHO=echo INCLUDE_EXTS=.h .hh .hpp .rli .cuh .inc .i CPP_EXT=.cpp OBJ_SUF= @@ -61,7 +61,7 @@ JAVA_REQUIREMENTS=$DEFAULT_REQUIREMENTS PY_REQUIREMENTS=$DEFAULT_REQUIREMENTS BISON_DATA_DIR=contrib/tools/bison/bison/data CPP_BISON_SKELS=${BISON_DATA_DIR}/glr.cc ${BISON_DATA_DIR}/lalr1.cc ${BISON_DATA_DIR}/yacc.c ${BISON_DATA_DIR}/stack.hh ${BISON_DATA_DIR}/variant.hh ${BISON_DATA_DIR}/c++.m4 ${BISON_DATA_DIR}/c++-skel.m4 - + USE_PREBUILT_TOOLS=yes when ($OPENSOURCE == "yes") { USE_PREBUILT_TOOLS=no @@ -121,8 +121,8 @@ otherwise { YNDEXER_ARGS= YNDEXER_OUTPUT= } - -COVERAGE_FLAGS= + +COVERAGE_FLAGS= EXTRA_OUTPUT= when ($CLANG == "yes" || $GCC == "yes") { when ($BUILD_TYPE == "COVERAGE" || $GCOV_COVERAGE) { @@ -130,16 +130,16 @@ when ($CLANG == "yes" || $GCC == "yes") { EXTRA_OUTPUT=${output;noauto;hide;suf=${OBJ_CROSS_SUF}${COMPILE_OUT_SUFFIX}.gcno:SRC} } } - + when ($OS_CYGWIN == "yes") { CFLAGS+=-D_LDBL_EQ_DBL=1 -U__STRICT_ANSI__ USE_ASMLIB=no FSTACK= } -CFLAGS+=$COVERAGE_FLAGS -LDFLAGS+=$COVERAGE_FLAGS - +CFLAGS+=$COVERAGE_FLAGS +LDFLAGS+=$COVERAGE_FLAGS + CHECKFLAG= LEX_FLAGS= NO_MAPREDUCE= @@ -196,9 +196,9 @@ when ($OS_LINUX && $OS_SDK == "ubuntu-14" && $MUSL != "yes") { USE_UBUNTU_COMPATIBILITY=yes } -USE_ARCADIA_PYTHON=yes +USE_ARCADIA_PYTHON=yes USE_ARCADIA_LIBM=no -USE_EAT_MY_DATA=no +USE_EAT_MY_DATA=no HAVE_MKL= when ($HAVE_MKL == "") { @@ -284,7 +284,7 @@ when ($PERLC == "yes") { XSUBPPFLAGS= # tag:tool-specific -ARCH_TOOL=${tool:"tools/archiver"} +ARCH_TOOL=${tool:"tools/archiver"} # tag:tool-specific tag:proto PROTOC=${tool:"contrib/tools/protoc"} @@ -294,9 +294,9 @@ PROTOBUF_PATH=${ARCADIA_ROOT}/contrib/libs/protobuf/src USE_VANILLA_PROTOC=no # tag:tool-specific -FML_TOOL=${tool:"tools/relev_fml_codegen"} -FML_UNUSED_TOOL=${tool:"tools/relev_fml_unused"} -LUA_TOOL=${tool:"tools/lua"} +FML_TOOL=${tool:"tools/relev_fml_codegen"} +FML_UNUSED_TOOL=${tool:"tools/relev_fml_unused"} +LUA_TOOL=${tool:"tools/lua"} ENUM_PARSER_TOOL=${tool:"tools/enum_parser/enum_parser"} # tag:python-specific tag:cython-specific @@ -485,8 +485,8 @@ PY_PROTOS_FOR=no BUILD_PROTO_AS_EVLOG=no # tag:proto -PROTO_NAMESPACE= - +PROTO_NAMESPACE= + # tag:proto tag:python-specific GEN_PY_PROTOS=$YMAKE_PYTHON ${input:"build/scripts/gen_py_protos.py"} @@ -807,7 +807,7 @@ macro GO_PROTO_PLUGIN(NAME, EXT, TOOL, DEPS[]) { macro _GO_PROTOC_PLUGIN_ARGS_BASE_IMPL(Name, Tool, PLUGINS...) { .CMD=--plugin=protoc-gen-${Name}=${tool:Tool} --${Name}_out=${join=|:PLUGINS}:${ARCADIA_BUILD_ROOT}/$PROTO_NAMESPACE } - + # tag:go-specific tag:proto macro _GO_PROTOC_PLUGIN_ARGS_BASE_PROXY(Name, Tool, MAP[], PLUGINS[]) { .CMD=$_GO_PROTOC_PLUGIN_ARGS_BASE_IMPL($Name $Tool $MAP ${pre=plugins=:PLUGINS}) @@ -1177,13 +1177,13 @@ module _BASE_UNIT: _BARE_UNIT { when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") { CFLAGS+=-fsanitize=$SANITIZER_TYPE -D${SANITIZER_TYPE}_sanitizer_enabled $SANITIZER_CFLAGS -fno-omit-frame-pointer - LDFLAGS+=-fsanitize=$SANITIZER_TYPE + LDFLAGS+=-fsanitize=$SANITIZER_TYPE SANITIZER_DEFINED=yes - - when ($CLANG) { - CFLAGS+=-fsanitize-blacklist=${input:"build/sanitize-blacklist.txt"} - LDFLAGS+=-fsanitize-blacklist=${input:"build/sanitize-blacklist.txt"} - } + + when ($CLANG) { + CFLAGS+=-fsanitize-blacklist=${input:"build/sanitize-blacklist.txt"} + LDFLAGS+=-fsanitize-blacklist=${input:"build/sanitize-blacklist.txt"} + } } when ($SANITIZE_COVERAGE && $SANITIZE_COVERAGE != "no") { @@ -1847,10 +1847,10 @@ module _BASE_UNITTEST: _BASE_PROGRAM { UNITTEST_SEM=$CPP_PROGRAM_SEM && add_test NAME $REALPRJNAME COMMAND $REALPRJNAME # tag:test ### @usage: UNITTEST([name]) -### +### ### Unit test module based on library/cpp/testing/unittest. ### It is recommended not to specify the name. -### +### ### Documentation: https://wiki.yandex-team.ru/yatool/test/#opisanievya.make1 module UNITTEST: _BASE_UNITTEST { .SEM=UNITTEST_SEM @@ -1890,18 +1890,18 @@ module GTEST: _BASE_UNITTEST { SET(MODULE_LANG CPP) } -USE_AFL=no - +USE_AFL=no + # tag:fuzzing ### @usage: FUZZ() -### +### ### In order to start using Fuzzing in Arcadia, you need to create a FUZZ module with the implementation of the function LLVMFuzzerTestOneInput(). ### This module should be reachable by RECURSE from /autocheck project in order for the corpus to be regularly updated. ### AFL and Libfuzzer are supported in Arcadia via a single interface, but the automatic fuzzing still works only through Libfuzzer. -### +### ### Example: https://a.yandex-team.ru/arc/trunk/arcadia/contrib/libs/re2/re2/fuzzing/re2_fuzzer.cc?rev=2919463#L58 ### -### Documentation: https://wiki.yandex-team.ru/yatool/fuzzing/ +### Documentation: https://wiki.yandex-team.ru/yatool/fuzzing/ module FUZZ: _BASE_PROGRAM { .NODE_TYPE=Program .FINAL_TARGET=no @@ -1920,8 +1920,8 @@ module FUZZ: _BASE_PROGRAM { } ADD_YTEST($MODULE_PREFIX$REALPRJNAME fuzz.test) -} - +} + # tag:ios-specific PACK_IOS_CMD= when ($OS_IOS && $BUILD_IOS_APP) { @@ -1974,17 +1974,17 @@ FUZZ_DICTS_VALUE= ### @usage: FUZZ_DICTS(path1 [path2...]) ### ### Allows you to specify dictionaries, relative to the root of Arcadia, which will be used in Fuzzing. -### Libfuzzer and AFL use a single syntax for dictionary descriptions. +### Libfuzzer and AFL use a single syntax for dictionary descriptions. ### Should only be used in FUZZ modules. -### -### Documentation: https://wiki.yandex-team.ru/yatool/fuzzing/ +### +### Documentation: https://wiki.yandex-team.ru/yatool/fuzzing/ macro FUZZ_DICTS(Data...) { SET_APPEND(FUZZ_DICTS_VALUE $Data) } FUZZ_OPTS_VALUE= ### @usage: FUZZ_OPTS(opt1 [Opt2...]) -### +### ### Overrides or adds options to the corpus mining and fuzzer run. ### Currently supported only Libfuzzer, so you should use the options for it. ### Should only be used in FUZZ modules. @@ -1996,7 +1996,7 @@ FUZZ_OPTS_VALUE= ### -rss_limit_mb=8192 ### ) ### -### Documentation: https://wiki.yandex-team.ru/yatool/fuzzing/ +### Documentation: https://wiki.yandex-team.ru/yatool/fuzzing/ macro FUZZ_OPTS(Data...) { SET_APPEND(FUZZ_OPTS_VALUE $Data) } @@ -2032,15 +2032,15 @@ macro DISABLE_DATA_VALIDATION() { # tag:test TEST_DATA_VALUE= -### @usage: DATA([path...]) -### +### @usage: DATA([path...]) +### ### Specifies the path to the data necessary test. ### Valid values are: arcadia/<path> , arcadia_tests_data/<path> and sbr://<resource_id>. ### In the latter case resource will be brought to the working directory of the test before it is started ### ### Used only inside TEST modules. -### -### Documentation: https://wiki.yandex-team.ru/yatool/test/#dannyeizrepozitorija +### +### Documentation: https://wiki.yandex-team.ru/yatool/test/#dannyeizrepozitorija macro DATA(Data...) { SET_APPEND(TEST_DATA_VALUE $Data) ADD_CHECK(check.data $Data) @@ -2049,10 +2049,10 @@ macro DATA(Data...) { # tag:test TEST_TAGS_VALUE= ### @usage: TAG ([tag...]) -### +### ### Each test can have one or more tags used to filter tests list for running. ### There are also special tags affecting test behaviour, for example ya:external, sb:ssd. -### +### ### Documentation: https://wiki.yandex-team.ru/yatool/test/#obshhieponjatija macro TAG(Tags...) { SET_APPEND(TEST_TAGS_VALUE $Tags) @@ -2061,9 +2061,9 @@ macro TAG(Tags...) { # tag:test TEST_REQUIREMENTS_VALUE= ### @usage: REQUIREMENTS([cpu:<count>] [disk_usage:<size>] [ram:<size>] [ram_disk:<size>] [container:<id>] [network:<restricted|full>] [dns:dns64]) -### -### Allows you to specify the requirements of the test. -### +### +### Allows you to specify the requirements of the test. +### ### Documentation about the Arcadia test system: https://wiki.yandex-team.ru/yatool/test/ macro REQUIREMENTS(Tags...) { SET_APPEND(TEST_REQUIREMENTS_VALUE $Tags) @@ -2119,8 +2119,8 @@ macro SKIP_TEST(Reason...) { # tag:test LINT_LEVEL_VALUE=extended -### @usage: NO_LINT() -### +### @usage: NO_LINT() +### ### Do not check for style files included in PY_SRCS, TEST_SRCS, JAVA_SRCS. macro NO_LINT() { SET(LINT_LEVEL_VALUE none) @@ -2171,7 +2171,7 @@ module PYTEST_BIN: _BASE_PYTEST { # tag:python-specific tag:test ### @usage: PY2TEST([name]) -### +### ### The test module for Python 2.x based on py.test ### ### This module is compatible only with PYTHON2-tagged modules and selects peers from multimodules accordingly. @@ -2262,15 +2262,15 @@ module GTEST_UGLY: _BASE_PROGRAM { # tag:test ### @usage: EXECTEST() -### +### ### Module definition of generic test that executes a binary. ### Use macro RUN to specify binary to run. -### +### ### @example: ### ### EXECTEST() ### OWNER(g:yatool) -### +### ### RUN( ### cat input.txt ### ) @@ -2282,9 +2282,9 @@ module GTEST_UGLY: _BASE_PROGRAM { ### ) ### TEST_CWD(devtools/ya/test/tests/exectest/data) ### END() -### +### ### More examples: https://wiki.yandex-team.ru/yatool/test/#exec-testy -### +### ### @see: [RUN()](#macro_RUN) module EXECTEST: _BARE_UNIT { .NODE_TYPE=Program @@ -2334,11 +2334,11 @@ when ($YA_DEV == "yes") { # tag:test -### @usage: UNITTEST_FOR(path/to/lib) +### @usage: UNITTEST_FOR(path/to/lib) ### ### Convenience extension of UNITTEST module. ### The UNINTTEST module with additional SRCDIR + ADDINCL + PEERDIR on path/to/lib. -### path/to/lib is the path to the directory with the LIBRARY project. +### path/to/lib is the path to the directory with the LIBRARY project. ### ### Documentation about the Arcadia test system: https://wiki.yandex-team.ru/yatool/test/ module UNITTEST_FOR: UNITTEST { @@ -2775,15 +2775,15 @@ module R_MODULE: DLL_UNIT { when ($DARWIN == "yes") { LDFLAGS+=-flat_namespace -dynamiclib } - + when ($MSVC == "yes" || $CYGWIN == "yes") { MODULE_PREFIX=lib MODULE_SUFFIX=.dll } ADDINCL(contrib/libs/r-lang) -} - +} + ### @usage: DYNAMIC_LIBRARY_FROM(Paths) ### ### Use specified libraries as sources of DLL @@ -2982,7 +2982,7 @@ module _DLL_COMPATIBLE_LIBRARY: LIBRARY { PACKED_PACKAGE_ARGS= PACKED_PACKAGE_EXT= -### @usage: PACK(archive_type) +### @usage: PACK(archive_type) ### ### When placed inside the PACKAGE module, packs the build results tree to the archive with specified extension. Currently supported extensions are `tar` and `tar.gz` ### @@ -2998,14 +2998,14 @@ macro PACKAGE_STRICT() { SET(PACKAGE_STRICT_VALUE yes) } -### @usage: PACKAGE(name) +### @usage: PACKAGE(name) ### ### Module collects what is described directly inside it, builds and collects all its transitively available PEERDIRs. ### As a result, build directory of the project gets the structure of the accessible part of Arcadia, where the build result of each PEERDIR is placed to relevant Arcadia subpath. ### The data can be optionally packed if macro PACK() is used. -### +### ### Is only used together with the macros FILES(), PEERDIR(), COPY(), FROM_SANDBOX(), RUN_PROGRAM or BUNDLE(). Don't use SRCS inside a PACKAGE. -### +### ### Documentation: https://wiki.yandex-team.ru/yatool/large-data/ ### ### @see: [PACK()](#macro_PACK) @@ -3061,7 +3061,7 @@ macro RESOLVE_PROTO() { ADDINCL(FOR proto $PROTOBUF_PATH) } -### @usage: UNION(name) +### @usage: UNION(name) ### ### Collection of PEERDIR dependencies, files and artifacts. ### UNION doesn't build its peers, just provides those to modules depending on it. @@ -4424,14 +4424,14 @@ REAL_SWIG_DLL_JAR_CMD=$GENERATE_VCS_JAVA_INFO_NODEP && ${cwd:ARCADIA_BUILD_ROOT} # tag:deprecated ### @usage: METAQUERY() #deprecated ### -### Project Definition - KIWI Meta query. (Objected) +### Project Definition - KIWI Meta query. (Objected) ### -### https://wiki.yandex-team.ru/robot/manual/kiwi/techdoc/design/metaquery/ +### https://wiki.yandex-team.ru/robot/manual/kiwi/techdoc/design/metaquery/ module METAQUERY: _BASE_UNIT { .ALL_INS_TO_OUT=yes .FINAL_TARGET=yes PRINT_MODULE_TYPE(METAQUERY $MODULE_PREFIX$REALPRJNAME) -} +} ARGS_DELIM="MACRO_CALLS_DELIM" @@ -4477,11 +4477,11 @@ macro USE_ERROR_PRONE() { } TEST_CWD_VALUE= -### @usage: TEST_CWD(path) -### +### @usage: TEST_CWD(path) +### ### Defines working directory for test runs. Othen used in conjunction with DATA() macro. ### Is only used inside of the TEST modules. -### +### ### Documentation: https://wiki.yandex-team.ru/yatool/test/ macro TEST_CWD(Arg) { SET(TEST_CWD_VALUE $Arg) @@ -4683,26 +4683,26 @@ macro MAVEN_GROUP_ID(Arg) { # tag:java-specific ANNOTATION_PROCESSOR_VALUE= -### @usage: ANNOTATION_PROCESSOR(processors...) +### @usage: ANNOTATION_PROCESSOR(processors...) ### -### The macro is in development. +### The macro is in development. ### Used to specify annotation processors to build JAVA_PROGRAM() and JAVA_LIBRARY(). macro ANNOTATION_PROCESSOR(Args...) { SET_APPEND(ANNOTATION_PROCESSOR_VALUE $ARGS_DELIM $Args) } EXCLUDE_VALUE= -### EXCLUDE(prefixes) +### EXCLUDE(prefixes) ### ### The macro is in development. -### Specifies which libraries should be excluded from the classpath. +### Specifies which libraries should be excluded from the classpath. macro EXCLUDE(Args...) { SET_APPEND(EXCLUDE_VALUE $ARGS_DELIM $Args) } # tag:java-specific JAVA_SRCS_VALUE= -### @usage: JAVA_SRCS(srcs) +### @usage: JAVA_SRCS(srcs) ### ### Specify java source files and resources. A macro can be contained in any of four java modules. ### Keywords: @@ -5424,14 +5424,14 @@ macro ADD_COMPILABLE_TRANSLIT(TranslitTable, NGrams, Name, Options...) { # tag:python-specific tag:proto tag:deprecated ### @usage GENERATE_PY_PROTOS(ProtoFiles...) # deprecated -### +### ### Generate python bindings for protobuf files. ### Macro is obsolete and not recommended for use! macro GENERATE_PY_PROTOS(FILES...) { foreach (FILE : $FILES) { _PY_PROTO_CMD($FILE) } -} +} # tag:python-specific tag:proto macro _GENERATE_PY_PROTOS_INTERNAL(FILES...) { @@ -5483,9 +5483,9 @@ macro _BUNDLE_TARGET(Target, Destination) { ### @usage: TIMEOUT(TIMEOUT) ### -### Sets a timeout on test execution -### -### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ +### Sets a timeout on test execution +### +### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ macro TIMEOUT(Time) { SET(TEST_TIMEOUT $Time) } @@ -5501,70 +5501,70 @@ macro VALIDATE_DATA_RESTART(Ext) { } # tag:test -TEST_FORK_MODE=none +TEST_FORK_MODE=none TEST_PARTITION=SEQUENTIAL - + # tag:test -### @usage: FORK_TESTS() -### +### @usage: FORK_TESTS() +### ### Splits a test run on chunks by test classes. -### The number of chunks can be overridden using the macro SPLIT_FACTOR. -### +### The number of chunks can be overridden using the macro SPLIT_FACTOR. +### ### Allows to run tests in parallel. Supported in UNITTEST, JTEST/JUNIT5 and PY2TEST/PY3TEST modules. -### -### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ +### +### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ macro FORK_TESTS(MODE...) { SET(TEST_FORK_MODE tests) SET(TEST_PARTITION $MODE) -} - +} + # tag:test -### @usage: FORK_SUBTESTS() -### -### Splits the test run in chunks on subtests. -### The number of chunks can be overridden using the macro SPLIT_FACTOR. -### +### @usage: FORK_SUBTESTS() +### +### Splits the test run in chunks on subtests. +### The number of chunks can be overridden using the macro SPLIT_FACTOR. +### ### Allows to run tests in parallel. Supported in UNITTEST, JTEST/JUNIT5 and PY2TEST/PY3TEST modules. -### -### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ +### +### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ macro FORK_SUBTESTS(MODE...) { SET(TEST_FORK_MODE subtests) SET(TEST_PARTITION $MODE) -} - +} + # tag:test -### @usage: SPLIT_FACTOR(x) -### +### @usage: SPLIT_FACTOR(x) +### ### Sets the number of chunks for parallel run tests when used in test module with FORK_TESTS() or FORK_SUBTESTS(). ### If none of those is specified this macro implies FORK_TESTS(). ### -### Supports C++ ut and PyTest. -### -### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ +### Supports C++ ut and PyTest. +### +### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ macro SPLIT_FACTOR(Factor) { SET(TEST_SPLIT_FACTOR $Factor) -} - +} + # tag:test FORK_TEST_FILES_MODE= -### @usage: FORK_TEST_FILES() -### +### @usage: FORK_TEST_FILES() +### ### Only for PY2TEST and PY3TEST: splits a file executable with the tests on chunks in the files listed in TEST_SRCS -### Compatible with FORK_(SUB)TESTS. -### -### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ +### Compatible with FORK_(SUB)TESTS. +### +### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ macro FORK_TEST_FILES() { SET(FORK_TEST_FILES_MODE on) } # tag:test TEST_SIZE_NAME=SMALL -### @usage: SIZE(SMALL/MEDIUM/LARGE) -### +### @usage: SIZE(SMALL/MEDIUM/LARGE) +### ### Set the 'size' for the test. Each 'size' has own set of resrtictions, SMALL bein the most restricted and LARGE being the list. ### See documentation on test system for more details. ### -### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ +### Documentation about the system test: https://wiki.yandex-team.ru/yatool/test/ macro SIZE(Type) { SET(TEST_SIZE_NAME $Type) } @@ -6144,14 +6144,14 @@ macro SRC(FILE, FLAGS...) { } ### @usage: SRCS(<[GLOBAL] File> ...) -### +### ### Source files of the project. Files are built according to their extension and put int module output or fed to ultimate PROGRAM/DLL depending on GLOBAL presence. -### Arcadia Paths from the root and is relative to the project's LIST are supported -### +### Arcadia Paths from the root and is relative to the project's LIST are supported +### ### GLOBAL marks next file as direct input to link phase of the program/shared library project built into. This prevents symbols of the file to be excluded by linker as unused. ### The scope of the GLOBAL keyword is the following file (that is, in the case of SRCS(GLOBAL foo.cpp bar.cpp) global will be only foo.cpp) -### -### @example: +### +### @example: ### ### LIBRARY(test_global) ### SRCS(GLOBAL foo.cpp) @@ -6272,7 +6272,7 @@ macro SRC_CPP_AVX2(FILE, FLAGS...) { # tag:python-processing tag:cython # TODO: use it in [.pyx] cmd ### @usage: BUILDWITH_CYTHON_CPP(Src Options...) -### +### ### Generates .cpp file from .pyx. macro BUILDWITH_CYTHON_CPP(Src, Options...) { .CMD=$RUN_CYTHON_SCRIPT $CYTHON_OPTIONS ${Options} --cplus ${CYTHON_CPP_OUTPUT_INCLUDES} ${pre=-I:_CYTHON__INCLUDE} ${input:Src} -o ${output;tobindir:Src.cpp} $CYTHON_OUTPUT_INCLUDES ${requirements;hide:PY_REQUIREMENTS} ${kv;hide:"p CY"} ${kv;hide:"pc yellow"} @@ -6291,7 +6291,7 @@ macro _BUILDWITH_CYTHON_CPP_DEP(Src, Dep, Options...) { # tag:python-processing tag:cython ### @usage: BUILDWITH_CYTHON_C(Src Options...) -### +### ### Generates .c file from .pyx. macro BUILDWITH_CYTHON_C(Src, Options...) { .CMD=$RUN_CYTHON_SCRIPT $CYTHON_OPTIONS ${Options} ${pre=-I:_CYTHON__INCLUDE} ${input:Src} -o ${output;tobindir:Src.c} $CYTHON_OUTPUT_INCLUDES ${requirements;hide:PY_REQUIREMENTS} ${kv;hide:"p CY"} ${kv;hide:"pc yellow"} @@ -6355,10 +6355,10 @@ macro BUILDWITH_RAGEL6(Src, Options...) { # tag:python-processing tag:internal # TODO: use it in [.pyx] cmd ### @usage: _PY_REGISTER() # internal -### +### ### Register Python 2.x module in internal resource file system. Arcadia Python 2.x importer will be retrieve these on import directive. ### -### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/pysrcs/#makrospyregister +### Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/pysrcs/#makrospyregister macro _PY_REGISTER(Func) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/gen_py_reg.py"} $Func ${output;noauto:Func.reg.cpp} ${requirements;hide:PY_REQUIREMENTS} ${kv;hide:"p PY"} ${kv;hide:"pc yellow"} SRCS(GLOBAL $Func.reg.cpp) @@ -6395,11 +6395,11 @@ macro _PY3_COMPILE_BYTECODE(SrcX, Src, Dst) { .CMD=${env:"PYTHONHASHSEED=0"} ${tool:"contrib/tools/python3/pycc"} $SrcX ${input:Src} ${output;noauto:Dst.yapyc3} ${requirements;hide:PY_REQUIREMENTS} ${kv;hide:"p PY"} ${kv;hide:"pc yellow"} } -### @usage: ARCHIVE_ASM(NAME archive_name files...) -### +### @usage: ARCHIVE_ASM(NAME archive_name files...) +### ### Similar to the macro ARCHIVE, but: ### 1. works faster and it is better to use for large files. -### 2. Different syntax (see examples in codesearch or users/pg/tests/archive_test) +### 2. Different syntax (see examples in codesearch or users/pg/tests/archive_test) macro ARCHIVE_ASM(NAME="", DONTCOMPRESS?"-p":"", REQUIREMENTS[], Files...) { .CMD=$ARCH_TOOL -q $DONTCOMPRESS ${input;join=\: :Files}: -o ${output;chksum;suf=$OBJ_SUF.rodata:NAME} ${requirements;hide:REQUIREMENTS} ${kv;hide:"p AR"} ${kv;hide:"pc light-cyan"} } @@ -6417,10 +6417,10 @@ macro PIRE_INLINE(FILES...) { } ### @usage: ARCHIVE(archive_name [DONT_COMPRESS] files...) -### +### ### Add arbitrary data to a modules. Unlike RESOURCE macro the result should be futher processed by othet macros in the module. ### -### Example: https://wiki.yandex-team.ru/yatool/howtowriteyamakefiles/#a1ispolzujjtekomanduarchive +### Example: https://wiki.yandex-team.ru/yatool/howtowriteyamakefiles/#a1ispolzujjtekomanduarchive macro ARCHIVE(NAME="", DONTCOMPRESS?"-p":"", REQUIREMENTS[], Files...) { .CMD=$ARCH_TOOL -q -x $DONTCOMPRESS ${input;join=\: :Files}: -o ${output;chksum;addincl;noauto:NAME} ${requirements;hide:REQUIREMENTS} ${kv;hide:"p AR"} ${kv;hide:"pc light-red"} } @@ -6436,7 +6436,7 @@ macro ARCHIVE_BY_KEYS(NAME="", KEYS="", DONTCOMPRESS?"-p":"", REQUIREMENTS[], Fi } #scripts - + #special commands BUILDVERSION_SCRIPT=build/scripts/build_info_gen.py SVNVERSION_JAVA_MARKER=output-java-class @@ -6501,8 +6501,8 @@ macro CONFIGURE_FILE(Src, Dst) { .SEM=set_vasrs ${CFG_VARS} && configure_file $S/${input;rootrel:Src} $B/${output;rootrel:Dst} } -### @usage: BASE_CODEGEN(tool_path prefix) -### +### @usage: BASE_CODEGEN(tool_path prefix) +### ### Generator ${prefix}.cpp + ${prefix}.h files based on ${prefix}.in. macro BASE_CODEGEN(Tool, Prefix, Opts...) { .CMD=${tool:Tool} ${input:Prefix.in} ${output;nopath;noauto:Prefix.cpp} ${output;nopath:Prefix.h} $Opts ${kv;hide:"p BC"} ${kv;hide:"pc yellow"} @@ -6510,7 +6510,7 @@ macro BASE_CODEGEN(Tool, Prefix, Opts...) { # tag:internal ### @usage: _SPLIT_CODEGEN_BASE(tool prefix OUTS[] OUTPUT_INCLUDES[]) # internal -### +### ### Generator of a certain number .the. cpp file + one header .h file from .in. ### This is the call of the generator. Python macro SPLIT_CODEGEN() is defined in order to properly fill command outputs from OUT_NUM argument. macro _SPLIT_CODEGEN_BASE(Tool, Prefix, OUT[], OPTS[], OUTPUT_INCLUDES[]) { @@ -6549,7 +6549,7 @@ macro LDFLAGS(Flags...) { } # tag:flags -### @usage: CFLAGS([GLOBAL compiler_flag]* compiler_flags) +### @usage: CFLAGS([GLOBAL compiler_flag]* compiler_flags) ### ### Add the specified flags to the compilation command of C and C++ files. ### @params: GLOBAL - Propagates these flags to dependent projects @@ -6559,30 +6559,30 @@ macro CFLAGS(Flags...) { } # tag:flags -### @usage: MASMFLAGS(compiler flags) +### @usage: MASMFLAGS(compiler flags) ### Add the specified flags to the compilation command of .masm files. macro MASMFLAGS(Flags...) { SET_APPEND(MASMFLAGS $Flags) } # tag:flags -### @usage: CONLYFLAGS([GLOBAL compiler_flag]* compiler_flags) +### @usage: CONLYFLAGS([GLOBAL compiler_flag]* compiler_flags) ### Add the specified flags to the compilation command of .c (but not .cpp) files. -### @params: GLOBAL - Distributes these flags on dependent projects +### @params: GLOBAL - Distributes these flags on dependent projects macro CONLYFLAGS(Flags...) { SET_APPEND_WITH_GLOBAL(USER_CONLYFLAGS $Flags) } # tag:flags -### @usage: CXXFLAGS(compiler_flags) +### @usage: CXXFLAGS(compiler_flags) ### Add the specified flags to the compilation command of .cpp (but not .c) files. macro CXXFLAGS(Flags...) { SET_APPEND_WITH_GLOBAL(USER_CXXFLAGS $Flags) } # tag:flags -### @usage: CUDA_NVCC_FLAGS(compiler flags) -### Add the specified flags to the compile line .cu-files. +### @usage: CUDA_NVCC_FLAGS(compiler flags) +### Add the specified flags to the compile line .cu-files. macro CUDA_NVCC_FLAGS(Flags...) { SET_APPEND(CUDA_NVCC_FLAGS $Flags) } @@ -6616,7 +6616,7 @@ macro NO_COMPILER_WARNINGS() { ENABLE(NO_COMPILER_WARNINGS) } -### @usage: WERROR() +### @usage: WERROR() ### Consider warnings as errors in the current module. ### In the bright future will be removed, since WERROR is the default. ### Priorities: NO_COMPILER_WARNINGS > NO_WERROR > WERROR_MODE > WERROR. @@ -6624,7 +6624,7 @@ macro WERROR() { ENABLE(WERROR) } -### @usage: NO_WERROR() +### @usage: NO_WERROR() ### Override WERROR() behavior ### Priorities: NO_COMPILER_WARNINGS > NO_WERROR > WERROR_MODE > WERROR. macro NO_WERROR() { @@ -6661,9 +6661,9 @@ macro NO_UTIL() { ENABLE(NOUTIL) } -### @usage: NO_RUNTIME() -### -### This macro: +### @usage: NO_RUNTIME() +### +### This macro: ### 1. Sets the ENABLE(NOUTIL) + DISABLE(USE_INTERNAL_STL); ### 2. If the project that contains the macro NO_RUNTIME(), peerdir-it project does not contain NO_RUNTIME() => Warning. ### Note: use this with care. Arcadia STL most likely will be linked into executable anyway, so using STL headers/functions/classes @@ -6674,7 +6674,7 @@ macro NO_RUNTIME() { ENABLE(NORUNTIME) } -### @usage: NO_LIBC() +### @usage: NO_LIBC() ### ### Exclude dependencies on C++ and C runtimes (including util, musl and libeatmydata). ### Note: use this with care. libc most likely will be linked into executable anyway, @@ -6727,8 +6727,8 @@ macro JOINSRC() { } # tag:sanitize -### @usage: NO_SANITIZE() -### +### @usage: NO_SANITIZE() +### ### Disable all sanitizers for the module. macro NO_SANITIZE() { DISABLE(SANITIZER_TYPE) @@ -6740,8 +6740,8 @@ macro NO_SANITIZE() { ### Disable lightweight coverage (-fsanitize-coverage) for the module. macro NO_SANITIZE_COVERAGE() { DISABLE(SANITIZE_COVERAGE) -} - +} + # tag:coverage ### @usage: NO_CLANG_COVERAGE() ### @@ -6808,7 +6808,7 @@ RELEV_FML_CODEGEN_INCLS=${output_include;hide:"kernel/relevfml/relev_fml.h"} ${o ### ### Create serialization support for enumeration members defined in the header (String <-> Enum conversions) and compile it into the module. ### -### Documentation: https://wiki.yandex-team.ru/yatool/HowToWriteYaMakeFiles/ +### Documentation: https://wiki.yandex-team.ru/yatool/HowToWriteYaMakeFiles/ macro GENERATE_ENUM_SERIALIZATION(File) { .CMD=$ENUM_PARSER_TOOL ${input:File} --include-path ${input;rootrel:File} --output ${output;chksum;suf=_serialized.cpp:File} ${output_include;hide:File} ${output_include;hide:"util/generic/serialized_enum.h"} ${kv;hide:"p EN"} ${kv;hide:"pc yellow"} .SEM=generate_enum_serilization ${input:File} ${output;hide;suf=_serialized.o:File} INCLUDE_HEADERS ${input;rootrel:File} ${tool;hide:"tools/enum_parser/enum_parser/bin"} @@ -6827,8 +6827,8 @@ macro GENERATE_ENUM_SERIALIZATION_WITH_HEADER(File) { PEERDIR(tools/enum_parser/enum_serialization_runtime) } -### @usage: DEB_VERSION(File) -### +### @usage: DEB_VERSION(File) +### ### Creates a header file DebianVersion.h define the DEBIAN_VERSION taken from the File. macro DEB_VERSION(File) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/mkver.py"} ${input:File} ${output;stdout:"DebianVersion.h"} ${kv;hide:"p CL"} ${kv;hide:"pc yellow"} @@ -6911,9 +6911,9 @@ macro NO_NEED_CHECK(Flags...) { # tag:deprecated ### @usage: NEED_REVIEW() # deprecated ### -### Mark the project as needing review. +### Mark the project as needing review. ### Reviewers are listed in the macro OWNER. The use of this macro is disabled by default. -### Details can be found here: https://clubs.at.yandex-team.ru/arcadia/6104 +### Details can be found here: https://clubs.at.yandex-team.ru/arcadia/6104 macro NEED_REVIEW(Flags...) { # TODO: FIXME ENABLE(UNUSED_MACRO) @@ -6924,9 +6924,9 @@ macro NEED_REVIEW(Flags...) { ### Specify version of a module. Currently unused by build system, only informative. macro VERSION(Flags...) { ENABLE(UNUSED_MACRO) -} - -DATAWORK_SCHEEME_EXPORT_FLAGS= +} + +DATAWORK_SCHEEME_EXPORT_FLAGS= when ($UNIX == "yes") { SCHEEME2_CFLAGS= -E -x c++ @@ -6937,34 +6937,34 @@ when ($WIN32 == "yes") { } SCHEEME2_STRUCT_INFO_FLAGS=-f "const static ui32 RecordSig" -u "RecordSig" --gcc44_no_typename --no_complex_overloaded_func_export -### @usage: GEN_SCHEEME2(scheeme_name from_file dependent_files...) -### -### Generates a C++ description for structure(contains the field RecordSig) in the specified file (and connected). -### +### @usage: GEN_SCHEEME2(scheeme_name from_file dependent_files...) +### +### Generates a C++ description for structure(contains the field RecordSig) in the specified file (and connected). +### ### 1. ${scheeme_name}.inc - the name of the generated file. -### 2. Use an environment variable - DATAWORK_SCHEEME_EXPORT_FLAGS that allows to specify flags to tools/structparser -### +### 2. Use an environment variable - DATAWORK_SCHEEME_EXPORT_FLAGS that allows to specify flags to tools/structparser +### ### @example: ### ### SET(DATAWORK_SCHEEME_EXPORT_FLAGS --final_only -m "::") ### -### all options are passed to structparser (in this example --final_only - do not export heirs with public base that contains the required field,,- m "::" only from the root namespace) +### all options are passed to structparser (in this example --final_only - do not export heirs with public base that contains the required field,,- m "::" only from the root namespace) ### sets in extra option -### +### ### @example: ### ### SET(EXTRACT_STRUCT_INFO_FLAGS -f \"const static ui32 RecordSig\" ### -u \"RecordSig\" -n${scheeme_name}SchemeInfo ----gcc44_no_typename no_complex_overloaded_func_export ### ${DATAWORK_SCHEEME_EXPORT_FLAGS}) ### -### for compatibility with C++ compiler and the external environment. +### for compatibility with C++ compiler and the external environment. ### See tools/structparser for more details. macro GEN_SCHEEME2(ScheemeName, FromFile) { .CMD=$CXX_COMPILER $C_FLAGS_PLATFORM -c ${tmp;stdout:FromFile.cph} $SCHEEME2_CFLAGS ${input:FromFile} ${pre=-I:_C__INCLUDE} $CXXFLAGS -Wno-error && ${tool:"tools/structparser"} -o ${output:ScheemeName.inc} -n N${ScheemeName}SchemeInfo $SCHEEME2_STRUCT_INFO_FLAGS $DATAWORK_SCHEEME_EXPORT_FLAGS ${tmp:FromFile.cph} ${output;stdout;noauto:ScheemeName.inc.log} ${kv;hide:"p SH"} ${kv;hide:"pc yellow"} } ### @usage: SYMLINK(from to) -### Add symlink +### Add symlink macro SYMLINK(From, To) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/symlink.py"} ${input;dirallowed:From} ${output;noauto:To} ${kv;hide:"p LN"} ${kv;hide:"pc light-cyan"} } @@ -6995,10 +6995,10 @@ macro RUN_PROGRAM(Tool, IN{input}[], OUT{output}[], OUT_NOAUTO{output}[], TOOL{t # tag:lua-specific ### @usage: LUA(script_path args... [CWD dir] [ENV key=value...] [TOOL tools...] [IN inputs...] [OUT[_NOAUTO] outputs...] [STDOUT[_NOAUTO] output] [OUTPUT_INCLUDES output_includes...][ REQUIREMENTS reqs]) -### +### ### Run a lua script. ### These macros are similar: RUN_PROGRAM, LUA, PYTHON. -### +### ### Parameters: ### - script_path - Path to the script.3 ### - args... - Program arguments. Relative paths listed in TOOL, IN, OUT, STDOUT become absolute. @@ -7019,10 +7019,10 @@ macro LUA(ScriptPath, IN{input}[], OUT{output}[], OUT_NOAUTO{output}[], TOOL{too # tag:python-specific ### @usage: PYTHON(script_path args... [CWD dir] [ENV key=value...] [TOOL tools...] [IN inputs...] [OUT[_NOAUTO] outputs...] [STDOUT[_NOAUTO] output] [OUTPUT_INCLUDES output_includes...] [REQUIREMENTS reqs]) -### +### ### Run a python script with $(PYTHON)/python built from devtools/huge_python. ### These macros are similar: RUN_PROGRAM, LUA, PYTHON. -### +### ### Parameters: ### - script_path - Path to the script. ### - args... - Program arguments. Relative paths listed in TOOL, IN, OUT, STDOUT become absolute. @@ -7127,8 +7127,8 @@ macro COMPILE_C_AS_CXX() { ### Compile files without debug info collection. macro NO_DEBUG_INFO() { SET(NO_DEBUGINFO yes) -} - +} + ### @usage: CTEMPLATE_VARNAMES(File) ### ### Generate File.varnames.h using contrib/libs/ctemplate/make_tpl_varnames_h @@ -7138,9 +7138,9 @@ macro CTEMPLATE_VARNAMES(File) { .CMD=${tool:"contrib/libs/ctemplate/make_tpl_varnames_h"} -f ${output;addincl;nopath;noallext:File.varnames.h} ${input:File} } -LLVM_OPTS= +LLVM_OPTS= CLANG_ROOT=$CLANG_RESOURCE_GLOBAL - + ### @usage: GENERATED_SRCS(srcs... PARSE_META_FROM cpp_srcs... [OUTPUT_INCLUDES output_includes...] [OPTIONS]) ### ### Generate sources using Jinja 2 template engine. @@ -7184,8 +7184,8 @@ macro CLANG_EMIT_AST_CXX(Input, Output, Opts...) { macro LLVM_COMPILE_CXX(Input, Output, Opts...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/clang_wrapper.py"} $WINDOWS $CLANG_ROOT/bin/clang++ ${pre=-I:_C__INCLUDE} $BC_CXXFLAGS $C_FLAGS_PLATFORM -Wno-unknown-warning-option $LLVM_OPTS -std=c++17 -fno-lto -emit-llvm -c ${input:Input} -o ${output;noauto:Output} $Opts ${kv;hide:"p BC"} ${kv;hide:"pc light-green"} PEERDIR(build/platform/clang) -} - +} + ### @usage: LLVM_COMPILE_C(Input Output Opts...) ### ### Emit LLVM bytecode from .c file. BC_CFLAGS, LLVM_OPTS and C_FLAGS_PLATFORM are passed in, while CFLAGS are not. @@ -7193,8 +7193,8 @@ macro LLVM_COMPILE_CXX(Input, Output, Opts...) { macro LLVM_COMPILE_C(Input, Output, Opts...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/clang_wrapper.py"} $WINDOWS $CLANG_ROOT/bin/clang ${pre=-I:_C__INCLUDE} $BC_CFLAGS $C_FLAGS_PLATFORM $LLVM_OPTS -fno-lto -emit-llvm -c ${input:Input} -o ${output;noauto:Output} $Opts ${kv;hide:"p BC"} ${kv;hide:"pc light-green"} PEERDIR(build/platform/clang) -} - +} + ### @usage: BPF(Input Output Opts...) ### ### Emit eBPF bytecode from .c file. @@ -7229,8 +7229,8 @@ macro LLVM_COMPILE_LL(Input, Output, Opts...) { macro LLVM_LINK(Output, Inputs...) { .CMD=$CLANG_ROOT/bin/llvm-link ${input:Inputs} -o ${output;noauto:Output} ${kv;hide:"p LD"} ${requirements;hide:LD_REQUIREMENTS} ${kv;hide:"pc light-red"} PEERDIR(build/platform/clang) -} - +} + ### @usage: LLVM_OPT(Input Output Opts...) ### ### Call llvm-opt with set of Opts on Input to produce Output. @@ -7238,12 +7238,12 @@ macro LLVM_LINK(Output, Inputs...) { macro LLVM_OPT(Input, Output, Opts...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/llvm_opt_wrapper.py"} $CLANG_ROOT/bin/opt ${input:Input} -o ${output;noauto:Output} $Opts ${kv;hide:"p OP"} ${kv;hide:"pc yellow"} PEERDIR(build/platform/clang) -} - +} + when ($NO_DEBUGINFO == "yes") { DEBUG_INFO_FLAGS= } - + when ($CLANG && $DEBUGINFO_LINES_ONLY == "yes" && $NO_DEBUGINFO != "yes") { DEBUG_INFO_FLAGS=-gline-tables-only } @@ -7257,17 +7257,17 @@ PACK_TGZ=${cwd:ARCADIA_BUILD_ROOT} tar -czf ${rootrel:OUTPUT} ${rootrel:INPUT} $ macro TOUCH(Outputs...) { .CMD=$YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${output:Outputs} } -TOUCH_UNIT=$YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${kv;hide:"p UN"} ${kv;hide:"pc light-cyan"} $TARGET +TOUCH_UNIT=$YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${kv;hide:"p UN"} ${kv;hide:"pc light-cyan"} $TARGET TOUCH_PACKAGE=$YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${kv;hide:"pc light-red"} $TARGET && $YMAKE_PYTHON ${input:"build/scripts/copy_to_dir.py"} --dest-dir $BINDIR --build-root $ARCADIA_BUILD_ROOT $PACKED_PACKAGE_ARGS $SRCS_GLOBAL $PEERS _P_PK=${kv;hide:"p PK"} TOUCH_PACKAGE_MF=$GENERATE_MF && $TOUCH_PACKAGE $_P_PK TOUCH_JAVA_UNIT=$YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${kv;hide:"java $CURDIR"} $TARGET NO_CHECK_IMPORTS_FOR_VALUE=None -### @usage: NO_CHECK_IMPORTS([patterns]) -### +### @usage: NO_CHECK_IMPORTS([patterns]) +### ### Do not run checks on imports of Python modules. -### Optional parameter mask patterns describes the names of the modules that do not need to check. +### Optional parameter mask patterns describes the names of the modules that do not need to check. macro NO_CHECK_IMPORTS(Masks...) { SET(NO_CHECK_IMPORTS_FOR_VALUE $Masks) } @@ -7380,8 +7380,8 @@ macro WHOLE_ARCHIVE(PEERS...) { ANDROID_SDK_ROOT=${ANDROID_SDK_RESOURCE_GLOBAL}/android_sdk -macro TASKLET() { - PEERDIR(tasklet/api) +macro TASKLET() { + PEERDIR(tasklet/api) # CPP CPP_PROTO_PLUGIN(tasklet_cpp tasklet/gen/cpp .tasklet.h) @@ -7391,16 +7391,16 @@ macro TASKLET() { #Java JAVA_PROTO_PLUGIN(tasket_java tasklet/gen/java DEPS tasklet/runtime/java/lib $JAVA_GRPC_STUB $JAVA_GRPC_PROTOBUF) -} - -TASKLET_REG_INCLUDES= \ - ${output_include;hide:"tasklet/runtime/lib/cpp_wrapper.h"} \ +} + +TASKLET_REG_INCLUDES= \ + ${output_include;hide:"tasklet/runtime/lib/cpp_wrapper.h"} \ ${output_include;hide:"tasklet/runtime/lib/go_wrapper.h"} \ ${output_include;hide:"tasklet/runtime/lib/py_wrapper.h"} \ ${output_include;hide:"tasklet/runtime/lib/js_wrapper.h"} \ - ${output_include;hide:"tasklet/runtime/lib/registry.h"} + ${output_include;hide:"tasklet/runtime/lib/registry.h"} -macro TASKLET_REG(Name, Lang, Impl, Includes...) { +macro TASKLET_REG(Name, Lang, Impl, Includes...) { PEERDIR(tasklet/domain sandbox/bin sandbox/taskbox/worker) when($Lang == "js") { @@ -7410,7 +7410,7 @@ macro TASKLET_REG(Name, Lang, Impl, Includes...) { } .CMD=$YMAKE_PYTHON ${input:"build/scripts/gen_tasklet_reg.py"} $Name -l $Lang -i $Impl ${output;noauto:Name.task.cpp} $Includes ${output_include;hide:Includes} $TASKLET_REG_INCLUDES ${kv;hide:"p TT"} ${kv;hide:"pc yellow"} - SRCS(GLOBAL $Name.task.cpp) + SRCS(GLOBAL $Name.task.cpp) } # TEMPORARY METHOD FOR EXTENDED REGISTRY SETUP diff --git a/build/ymake_conf.py b/build/ymake_conf.py index 30219eb85e..3908e93440 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -19,16 +19,16 @@ import tempfile import six logger = logging.getLogger(__name__ if __name__ != '__main__' else 'ymake_conf.py') - + def init_logger(verbose): logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) - - + + class DebugString(object): def __init__(self, get_string_func): self.get_string_func = get_string_func - + def __str__(self): return self.get_string_func() @@ -47,13 +47,13 @@ class Platform(object): self.name = name self.os = self._parse_os(os) self.arch = arch.lower() - + self.is_i386 = self.arch in ('i386', 'x86') self.is_i686 = self.arch == 'i686' self.is_x86 = self.is_i386 or self.is_i686 self.is_x86_64 = self.arch in ('x86_64', 'amd64') self.is_intel = self.is_x86 or self.is_x86_64 - + self.is_armv7 = self.arch in ('armv7', 'armv7a', 'armv7a_neon', 'arm', 'armv7a_cortex_a9', 'armv7ahf_cortex_a35', 'armv7ahf_cortex_a53') self.is_armv8 = self.arch in ('armv8', 'armv8a', 'arm64', 'aarch64', 'armv8a_cortex_a35', 'armv8a_cortex_a53') self.is_armv8m = self.arch in ('armv8m_cortex_m33',) @@ -208,8 +208,8 @@ class Platform(object): return 'cygwin' return os - - + + def which(prog): if os.path.exists(prog) and os.access(prog, os.X_OK): return prog @@ -230,15 +230,15 @@ def which(prog): p = os.path.join(dir_, prog + ext) if os.path.exists(p) and os.path.isfile(p) and os.access(p, os.X_OK): return p - + return None def get_stdout(command): stdout, code = get_stdout_and_code(command) return stdout if code == 0 else None - - + + def get_stdout_and_code(command): # noinspection PyBroadException try: @@ -248,7 +248,7 @@ def get_stdout_and_code(command): except Exception: return None, None - + def to_strings(o): if isinstance(o, (list, tuple)): for s in o: @@ -267,7 +267,7 @@ def to_strings(o): def emit(key, *value): print('{0}={1}'.format(key, ' '.join(to_strings(value)))) - + def emit_with_comment(comment, key, *value): print('# {}'.format(comment)) emit(key, *value) @@ -280,7 +280,7 @@ def emit_with_ignore_comment(key, *value): def append(key, *value): print('{0}+={1}'.format(key, ' '.join(to_strings(value)))) - + def emit_big(text): prefix = None first = True @@ -401,16 +401,16 @@ class Options(object): def __init__(self, argv): def parse_presets(raw_presets): presets = {} - for p in raw_presets: - toks = p.split('=', 1) + for p in raw_presets: + toks = p.split('=', 1) name = toks[0] value = toks[1] if len(toks) >= 2 else '' presets[name] = value return presets - parser = optparse.OptionParser(add_help_option=False) + parser = optparse.OptionParser(add_help_option=False) opt_group = optparse.OptionGroup(parser, 'Conf script options') - opt_group.add_option('--toolchain-params', dest='toolchain_params', action='store', help='Set toolchain params via file') + opt_group.add_option('--toolchain-params', dest='toolchain_params', action='store', help='Set toolchain params via file') opt_group.add_option('-D', '--preset', dest='presets', action='append', default=[], help='set or override presets') opt_group.add_option('-l', '--local-distbuild', dest='local_distbuild', action='store_true', default=False, help='conf for local distbuild') parser.add_option_group(opt_group) @@ -466,7 +466,7 @@ class Build(object): # TODO(somov): Удалить, когда перестанет использоваться. self.build_system = 'ymake' - + self.ignore_local_files = False dist_prefix = 'dist-' @@ -497,7 +497,7 @@ class Build(object): if self.pic: emit('PIC', 'yes') - + emit('COMPILER_ID', self.tc.type.upper()) if self.is_valgrind: @@ -513,7 +513,7 @@ class Build(object): linker.print_linker() self._print_other_settings(compiler) - + def _print_build_settings(self): emit('BUILD_TYPE', self.build_type.upper()) emit('BT_' + self.build_type.upper().replace('-', '_'), 'yes') @@ -741,21 +741,21 @@ class System(object): :type platform: Platform """ self.platform = platform - + def print_windows_target_const(self): # TODO(somov): Remove this variables, use generic OS/arch variables in makelists. emit('WINDOWS', 'yes') emit('WIN32', 'yes') if self.platform.is_64_bit == 64: emit('WIN64', 'yes') - + def print_nix_target_const(self): emit('JAVA_INCLUDE', '-I{0}/include -I{0}/include/{1}'.format('/usr/lib/jvm/default-java', self.platform.os_compat)) - + emit('UNIX', 'yes') emit('REALPRJNAME') emit('SONAME') - + @staticmethod def print_nix_host_const(): emit('WRITE_COMMAND', '/bin/echo', '-e') @@ -772,15 +772,15 @@ when (($USEMPROF == "yes") || ($USE_MPROF == "yes")) { C_SYSTEM_LIBRARIES_INTERCEPT+=-ldmalloc } ''') - + def print_target_settings(self): emit('TARGET_PLATFORM', self.platform.os_compat) emit('HARDWARE_ARCH', '32' if self.platform.is_32_bit else '64') emit('HARDWARE_TYPE', self.platform.arch) - + for variable in self.platform.arch_variables: emit(variable, 'yes') - + for variable in self.platform.os_variables: emit(variable, 'yes') @@ -843,7 +843,7 @@ class CompilerDetector(object): if stdout is None: return None - + vars_ = {} for line in stdout.split('\n'): parts = line.split('=', 1) @@ -863,7 +863,7 @@ class CompilerDetector(object): logger.debug('e=%s', os.environ) if c_compiler is None: raise ConfigureError('Custom compiler was requested but not specified') - + c_compiler_path = which(c_compiler) clang_vars = ['__clang_major__', '__clang_minor__', '__clang_patchlevel__'] @@ -920,7 +920,7 @@ class ToolchainOptions(object): """ self.host = build.host self.target = build.target - + tc_json = build.params logger.debug('Toolchain host %s', self.host) @@ -1016,7 +1016,7 @@ class ToolchainOptions(object): class GnuToolchainOptions(ToolchainOptions): def __init__(self, build, detector): super(GnuToolchainOptions, self).__init__(build, detector) - + self.ar = self.params.get('ar') self.ar_plugin = self.params.get('ar_plugin') self.inplace_tools = self.params.get('inplace_tools', False) @@ -1102,9 +1102,9 @@ class GnuToolchain(Toolchain): target = build.target self.c_flags_platform = list(tc.target_opt) - + self.default_os_sdk_root = get_os_sdk(target) - + self.env = self.tc.get_env() self.env_go = {} @@ -1937,7 +1937,7 @@ class LD(Linker): (not target.is_iossim and target.is_ios, '-Wl,-sdk_version,13.1'), (target.is_iossim, '-Wl,-sdk_version,14.5'), ]) - + if self.ld_sdk: self.ld_flags.append(self.ld_sdk) @@ -2174,7 +2174,7 @@ class LD(Linker): emit('MD5LIB', '-lcrypt') emit('LIBRESOLV', self.libresolv) emit('PROFFLAG', '-pg') - + class MSVCToolchainOptions(ToolchainOptions): def __init__(self, build, detector): @@ -2829,7 +2829,7 @@ class Ragel(object): self.rlgen_flags = [] self.ragel_flags = [] self.ragel6_flags = [] - + def configure_toolchain(self, build, compiler): if isinstance(compiler, MSVCCompiler): self.set_default_flags(optimized=False) diff --git a/contrib/libs/ya.make b/contrib/libs/ya.make index 9c4640fdcf..a97ba2abec 100644 --- a/contrib/libs/ya.make +++ b/contrib/libs/ya.make @@ -375,7 +375,7 @@ IF (OS_FREEBSD OR OS_LINUX) asmglibc httpd mod_perl2 - libeatmydata + libeatmydata ) ENDIF() diff --git a/contrib/python/ya.make b/contrib/python/ya.make index d01ced9f3a..ee415d1576 100644 --- a/contrib/python/ya.make +++ b/contrib/python/ya.make @@ -184,10 +184,10 @@ RECURSE( crontab cryptography cssselect - cssutils + cssutils curlify cx-Oracle - cycler + cycler cymem cytoolz czipfile @@ -750,7 +750,7 @@ RECURSE( plugincode plumbum ply - plyvel + plyvel polib portalocker portpicker diff --git a/contrib/tools/cython/Cython/Utility/Optimize.c b/contrib/tools/cython/Cython/Utility/Optimize.c index d18c9b78ec..ee9c953caa 100644 --- a/contrib/tools/cython/Cython/Utility/Optimize.c +++ b/contrib/tools/cython/Cython/Utility/Optimize.c @@ -1039,9 +1039,9 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED } {{endif}} // copied from floatobject.c in Py3.5: -// PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) +// PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) result = ((double)a) {{c_op}} (double)b; -// PyFPE_END_PROTECT(result) +// PyFPE_END_PROTECT(result) return PyFloat_FromDouble(result); {{endif}} } @@ -1178,7 +1178,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{else}} // copied from floatobject.c in Py3.5: {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check('b')}}{{endif}} -// PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) +// PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) {{if c_op == '%'}} result = fmod(a, b); if (result) @@ -1188,7 +1188,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{else}} result = a {{c_op}} b; {{endif}} -// PyFPE_END_PROTECT(result) +// PyFPE_END_PROTECT(result) return PyFloat_FromDouble(result); {{endif}} } diff --git a/contrib/tools/ragel6/cdcodegen.cpp b/contrib/tools/ragel6/cdcodegen.cpp index 9e784e3f58..da8e801a39 100644 --- a/contrib/tools/ragel6/cdcodegen.cpp +++ b/contrib/tools/ragel6/cdcodegen.cpp @@ -704,11 +704,11 @@ string CCodeGen::PTR_CONST_END() std::ostream &CCodeGen::OPEN_ARRAY( string type, string name ) { - out << "#if defined(__GNUC__)\n"; - out << "static __attribute__((used)) const " << type << " " << name << "[] = {\n"; - out << "#else\n"; + out << "#if defined(__GNUC__)\n"; + out << "static __attribute__((used)) const " << type << " " << name << "[] = {\n"; + out << "#else\n"; out << "static const " << type << " " << name << "[] = {\n"; - out << "#endif\n"; + out << "#endif\n"; return out; } diff --git a/library/cpp/cgiparam/fuzz/ya.make b/library/cpp/cgiparam/fuzz/ya.make index 8fb9d50d3b..e55c7e9634 100644 --- a/library/cpp/cgiparam/fuzz/ya.make +++ b/library/cpp/cgiparam/fuzz/ya.make @@ -1,4 +1,4 @@ -FUZZ() +FUZZ() OWNER( pg diff --git a/library/cpp/colorizer/colors.cpp b/library/cpp/colorizer/colors.cpp index decc5c9847..85c148893b 100644 --- a/library/cpp/colorizer/colors.cpp +++ b/library/cpp/colorizer/colors.cpp @@ -1,9 +1,9 @@ #include "colors.h" - + #include <util/stream/output.h> #include <util/generic/singleton.h> #include <util/system/env.h> - + #include <cstdlib> #if defined(_unix_) @@ -11,7 +11,7 @@ #endif using namespace NColorizer; - + namespace { constexpr TStringBuf ToStringBufC(NColorizer::EAnsiCode x) { switch(x) { @@ -171,30 +171,30 @@ TColors::TColors(FILE* f) { SetIsTTY(CalcIsTTY(f)); } - + TColors::TColors(bool ontty) : IsTTY_(true) { SetIsTTY(ontty); -} - +} + TStringBuf TColors::Reset() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::RESET) : ToStringBufC(EAnsiCode::INVALID); -} - +} + TStringBuf TColors::StyleLight() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::ST_LIGHT) : ToStringBufC(EAnsiCode::INVALID); -} +} TStringBuf TColors::StyleDark() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::ST_DARK) : ToStringBufC(EAnsiCode::INVALID); } TStringBuf TColors::StyleNormal() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::ST_NORMAL) : ToStringBufC(EAnsiCode::INVALID); } - + TStringBuf TColors::ItalicOn() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::ITALIC_ON) : ToStringBufC(EAnsiCode::INVALID); -} +} TStringBuf TColors::ItalicOff() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::ITALIC_OFF) : ToStringBufC(EAnsiCode::INVALID); } @@ -204,10 +204,10 @@ TStringBuf TColors::UnderlineOn() const noexcept { TStringBuf TColors::UnderlineOff() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::UNDERLINE_OFF) : ToStringBufC(EAnsiCode::INVALID); } - + TStringBuf TColors::ForeDefault() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::FG_DEFAULT) : ToStringBufC(EAnsiCode::INVALID); -} +} TStringBuf TColors::ForeBlack() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::FG_BLACK) : ToStringBufC(EAnsiCode::INVALID); } @@ -232,10 +232,10 @@ TStringBuf TColors::ForeCyan() const noexcept { TStringBuf TColors::ForeWhite() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::FG_WHITE) : ToStringBufC(EAnsiCode::INVALID); } - + TStringBuf TColors::BackDefault() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::BG_DEFAULT) : ToStringBufC(EAnsiCode::INVALID); -} +} TStringBuf TColors::BackBlack() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::BG_BLACK) : ToStringBufC(EAnsiCode::INVALID); } @@ -260,10 +260,10 @@ TStringBuf TColors::BackCyan() const noexcept { TStringBuf TColors::BackWhite() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::BG_WHITE) : ToStringBufC(EAnsiCode::INVALID); } - + TStringBuf TColors::Default() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::DEFAULT) : ToStringBufC(EAnsiCode::INVALID); -} +} TStringBuf TColors::Black() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::BLACK) : ToStringBufC(EAnsiCode::INVALID); } @@ -288,10 +288,10 @@ TStringBuf TColors::Cyan() const noexcept { TStringBuf TColors::White() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::WHITE) : ToStringBufC(EAnsiCode::INVALID); } - + TStringBuf TColors::LightDefault() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::LIGHT_DEFAULT) : ToStringBufC(EAnsiCode::INVALID); -} +} TStringBuf TColors::LightBlack() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::LIGHT_BLACK) : ToStringBufC(EAnsiCode::INVALID); } @@ -316,10 +316,10 @@ TStringBuf TColors::LightCyan() const noexcept { TStringBuf TColors::LightWhite() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::LIGHT_WHITE) : ToStringBufC(EAnsiCode::INVALID); } - + TStringBuf TColors::DarkDefault() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::DARK_DEFAULT) : ToStringBufC(EAnsiCode::INVALID); -} +} TStringBuf TColors::DarkBlack() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::DARK_BLACK) : ToStringBufC(EAnsiCode::INVALID); } @@ -344,47 +344,47 @@ TStringBuf TColors::DarkCyan() const noexcept { TStringBuf TColors::DarkWhite() const noexcept { return IsTTY() ? ToStringBufC(EAnsiCode::DARK_WHITE) : ToStringBufC(EAnsiCode::INVALID); } - + TStringBuf TColors::OldColor() const noexcept { return IsTTY() ? "\033[22;39m" : ""; -} - +} + TStringBuf TColors::BoldColor() const noexcept { return IsTTY() ? "\033[1m" : ""; -} - +} + TStringBuf TColors::BlackColor() const noexcept { return IsTTY() ? "\033[22;30m" : ""; -} - +} + TStringBuf TColors::BlueColor() const noexcept { return IsTTY() ? "\033[22;34m" : ""; -} - +} + TStringBuf TColors::GreenColor() const noexcept { return IsTTY() ? "\033[22;32m" : ""; -} - +} + TStringBuf TColors::CyanColor() const noexcept { return IsTTY() ? "\033[22;36m" : ""; -} - +} + TStringBuf TColors::RedColor() const noexcept { return IsTTY() ? "\033[22;31m" : ""; -} - +} + TStringBuf TColors::PurpleColor() const noexcept { return IsTTY() ? "\033[22;35m" : ""; -} - +} + TStringBuf TColors::BrownColor() const noexcept { return IsTTY() ? "\033[22;33m" : ""; -} - +} + TStringBuf TColors::LightGrayColor() const noexcept { return IsTTY() ? "\033[22;37m" : ""; -} - +} + TStringBuf TColors::DarkGrayColor() const noexcept { return IsTTY() ? "\033[1;30m" : ""; } @@ -404,15 +404,15 @@ TStringBuf TColors::LightCyanColor() const noexcept { TStringBuf TColors::LightRedColor() const noexcept { return IsTTY() ? "\033[1;31m" : ""; } - + TStringBuf TColors::LightPurpleColor() const noexcept { return IsTTY() ? "\033[1;35m" : ""; -} - +} + TStringBuf TColors::YellowColor() const noexcept { return IsTTY() ? "\033[1;33m" : ""; -} - +} + TStringBuf TColors::WhiteColor() const noexcept { return IsTTY() ? "\033[1;37m" : ""; } diff --git a/library/cpp/colorizer/colors.h b/library/cpp/colorizer/colors.h index 474a918994..2966c98375 100644 --- a/library/cpp/colorizer/colors.h +++ b/library/cpp/colorizer/colors.h @@ -1,10 +1,10 @@ -#pragma once +#pragma once #include "fwd.h" - + #include <util/generic/string.h> #include <util/generic/strbuf.h> - + #include <cstdio> namespace NColorizer { @@ -106,13 +106,13 @@ namespace NColorizer { public: explicit TColors(FILE* = stderr); explicit TColors(bool ontty); - + TStringBuf Reset() const noexcept; - + TStringBuf StyleLight() const noexcept; TStringBuf StyleDark() const noexcept; TStringBuf StyleNormal() const noexcept; - + TStringBuf ItalicOn() const noexcept; TStringBuf ItalicOff() const noexcept; TStringBuf UnderlineOn() const noexcept; @@ -195,7 +195,7 @@ namespace NColorizer { inline bool IsTTY() const noexcept { return IsTTY_; } - + inline void SetIsTTY(bool value) noexcept { IsTTY_ = value; } @@ -210,7 +210,7 @@ namespace NColorizer { private: bool IsTTY_; - }; + }; /// Singletone `TColors` instances for stderr/stdout. TColors& StdErr(); @@ -222,7 +222,7 @@ namespace NColorizer { /// Calculate total length of all ANSI escape codes in the text. size_t TotalAnsiEscapeCodeLen(TStringBuf text); -} +} TStringBuf ToStringBuf(NColorizer::EAnsiCode x); TString ToString(NColorizer::EAnsiCode x); diff --git a/library/cpp/colorizer/output.h b/library/cpp/colorizer/output.h index 99afbd3427..e2d5afcec4 100644 --- a/library/cpp/colorizer/output.h +++ b/library/cpp/colorizer/output.h @@ -25,19 +25,19 @@ namespace NColorizer { } DEF(Old) - DEF(Black) + DEF(Black) DEF(Green) - DEF(Cyan) + DEF(Cyan) DEF(Red) - DEF(Purple) - DEF(Brown) - DEF(LightGray) - DEF(DarkGray) - DEF(LightBlue) - DEF(LightGreen) - DEF(LightCyan) - DEF(LightRed) - DEF(LightPurple) + DEF(Purple) + DEF(Brown) + DEF(LightGray) + DEF(DarkGray) + DEF(LightBlue) + DEF(LightGreen) + DEF(LightCyan) + DEF(LightRed) + DEF(LightPurple) DEF(Yellow) DEF(White) diff --git a/library/cpp/http/fetch/ya.make b/library/cpp/http/fetch/ya.make index 7737127463..bda332045a 100644 --- a/library/cpp/http/fetch/ya.make +++ b/library/cpp/http/fetch/ya.make @@ -31,7 +31,7 @@ SRCS( GENERATE_ENUM_SERIALIZATION(httpheader.h) -SET(RAGEL6_FLAGS -CF1) +SET(RAGEL6_FLAGS -CF1) END() diff --git a/library/cpp/http/io/fuzz/ya.make b/library/cpp/http/io/fuzz/ya.make index 8b3ccb1969..0a901fe0d4 100644 --- a/library/cpp/http/io/fuzz/ya.make +++ b/library/cpp/http/io/fuzz/ya.make @@ -1,4 +1,4 @@ -FUZZ() +FUZZ() OWNER( pg diff --git a/library/cpp/json/fuzzy_test/ya.make b/library/cpp/json/fuzzy_test/ya.make index ff50bc1f62..aec610841f 100644 --- a/library/cpp/json/fuzzy_test/ya.make +++ b/library/cpp/json/fuzzy_test/ya.make @@ -1,4 +1,4 @@ -FUZZ() +FUZZ() OWNER(pg) diff --git a/library/cpp/resource/registry.cpp b/library/cpp/resource/registry.cpp index 66001c4769..8f5a718a81 100644 --- a/library/cpp/resource/registry.cpp +++ b/library/cpp/resource/registry.cpp @@ -40,7 +40,7 @@ namespace { " new size: %zu.", TString{key}.Quote().c_str(), vsize, dsize); } - } + } } else { D_.push_back(TDescriptor(key, data)); (*this)[key] = &D_.back(); diff --git a/library/cpp/testing/unittest/utmain.cpp b/library/cpp/testing/unittest/utmain.cpp index 305bc6b40f..55564fd3d3 100644 --- a/library/cpp/testing/unittest/utmain.cpp +++ b/library/cpp/testing/unittest/utmain.cpp @@ -542,28 +542,28 @@ public: TEnumeratingProcessor(bool verbose, IOutputStream& stream) noexcept : Verbose_(verbose) , Stream_(stream) - { - } - + { + } + ~TEnumeratingProcessor() override { } bool CheckAccess(TString name, size_t /*num*/) override { - if (Verbose_) { - return true; - } else { + if (Verbose_) { + return true; + } else { Stream_ << name << "\n"; - return false; - } - } - + return false; + } + } + bool CheckAccessTest(TString suite, const char* name) override { Stream_ << suite << "::" << name << "\n"; return false; } - -private: - bool Verbose_; + +private: + bool Verbose_; IOutputStream& Stream_; }; diff --git a/library/python/filelock/__init__.py b/library/python/filelock/__init__.py index f81ff67f37..1d0fe9b535 100644 --- a/library/python/filelock/__init__.py +++ b/library/python/filelock/__init__.py @@ -41,7 +41,7 @@ class _NixFileLock(AbstractFileLock): super(_NixFileLock, self).__init__(path) from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_NB self._locker = lambda lock, blocking: flock(lock, LOCK_EX if blocking else LOCK_EX | LOCK_NB) - self._unlocker = lambda lock: flock(lock, LOCK_UN) + self._unlocker = lambda lock: flock(lock, LOCK_UN) self._lock = open(self.path, 'a') set_close_on_exec(self._lock) diff --git a/library/python/fs/__init__.py b/library/python/fs/__init__.py index b1b7cde079..f5d2a2b2ce 100644 --- a/library/python/fs/__init__.py +++ b/library/python/fs/__init__.py @@ -1,21 +1,21 @@ # coding: utf-8 import codecs -import errno -import logging -import os +import errno +import logging +import os import random -import shutil +import shutil import six import stat import sys - + import library.python.func import library.python.strings import library.python.windows - + logger = logging.getLogger(__name__) - + try: WindowsError @@ -40,12 +40,12 @@ class CustomFsError(OSError): # Throws OSError @errorfix_win def ensure_dir(path): - try: - os.makedirs(path) - except OSError as e: + try: + os.makedirs(path) + except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(path): - raise - + raise + # Directories creation # If dst is already exists and is a directory - does nothing @@ -54,9 +54,9 @@ def ensure_dir(path): @errorfix_win def create_dirs(path): ensure_dir(path) - return path - - + return path + + # Atomic file/directory move (rename) # Doesn't guarantee dst replacement # Atomic if no device boundaries are crossed @@ -72,8 +72,8 @@ def create_dirs(path): @library.python.windows.diehard(library.python.windows.RETRIABLE_FILE_ERRORS, tries=_diehard_win_tries) def move(src, dst): os.rename(src, dst) - - + + # Atomic replacing file move (rename) # Replaces dst if exists and not a dir # Doesn't guarantee dst dir replacement @@ -372,94 +372,94 @@ def copytree3( ignore_dangling_symlinks=False, dirs_exist_ok=False, ): - """Recursively copy a directory tree. - - The copytree3 is a port of shutil.copytree function from python-3.2. - It has additional useful parameters and may be helpful while we are - on python-2.x. It has to be removed as soon as we have moved to - python-3.2 or higher. - - The destination directory must not already exist. - If exception(s) occur, an Error is raised with a list of reasons. - - If the optional symlinks flag is true, symbolic links in the - source tree result in symbolic links in the destination tree; if - it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't - exist, an exception will be added in the list of errors raised in - an Error exception at the end of the copy process. - - You can set the optional ignore_dangling_symlinks flag to true if you - want to silence this exception. Notice that this has no effect on - platforms that don't support os.symlink. - - The optional ignore argument is a callable. If given, it - is called with the `src` parameter, which is the directory - being visited by copytree3(), and `names` which is the list of - `src` contents, as returned by os.listdir(): - - callable(src, names) -> ignored_names - - Since copytree3() is called recursively, the callable will be - called once for each directory that is copied. It returns a - list of names relative to the `src` directory that should - not be copied. - - The optional copy_function argument is a callable that will be used - to copy each file. It will be called with the source path and the - destination path as arguments. By default, copy2() is used, but any - function that supports the same signature (like copy()) can be used. - - """ - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - + """Recursively copy a directory tree. + + The copytree3 is a port of shutil.copytree function from python-3.2. + It has additional useful parameters and may be helpful while we are + on python-2.x. It has to be removed as soon as we have moved to + python-3.2 or higher. + + The destination directory must not already exist. + If exception(s) occur, an Error is raised with a list of reasons. + + If the optional symlinks flag is true, symbolic links in the + source tree result in symbolic links in the destination tree; if + it is false, the contents of the files pointed to by symbolic + links are copied. If the file pointed by the symlink doesn't + exist, an exception will be added in the list of errors raised in + an Error exception at the end of the copy process. + + You can set the optional ignore_dangling_symlinks flag to true if you + want to silence this exception. Notice that this has no effect on + platforms that don't support os.symlink. + + The optional ignore argument is a callable. If given, it + is called with the `src` parameter, which is the directory + being visited by copytree3(), and `names` which is the list of + `src` contents, as returned by os.listdir(): + + callable(src, names) -> ignored_names + + Since copytree3() is called recursively, the callable will be + called once for each directory that is copied. It returns a + list of names relative to the `src` directory that should + not be copied. + + The optional copy_function argument is a callable that will be used + to copy each file. It will be called with the source path and the + destination path as arguments. By default, copy2() is used, but any + function that supports the same signature (like copy()) can be used. + + """ + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + if not (dirs_exist_ok and os.path.isdir(dst)): os.makedirs(dst) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if os.path.islink(srcname): - linkto = os.readlink(srcname) - if symlinks: - # We can't just leave it to `copy_function` because legacy - # code with a custom `copy_function` may rely on copytree3 - # doing the right thing. - os.symlink(linkto, dstname) - else: - # ignore dangling symlink if the flag is on - if not os.path.exists(linkto) and ignore_dangling_symlinks: - continue - # otherwise let the copy occurs. copy2 will raise an error - copy_function(srcname, dstname) - elif os.path.isdir(srcname): + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if os.path.islink(srcname): + linkto = os.readlink(srcname) + if symlinks: + # We can't just leave it to `copy_function` because legacy + # code with a custom `copy_function` may rely on copytree3 + # doing the right thing. + os.symlink(linkto, dstname) + else: + # ignore dangling symlink if the flag is on + if not os.path.exists(linkto) and ignore_dangling_symlinks: + continue + # otherwise let the copy occurs. copy2 will raise an error + copy_function(srcname, dstname) + elif os.path.isdir(srcname): copytree3(srcname, dstname, symlinks, ignore, copy_function, dirs_exist_ok=dirs_exist_ok) - else: - # Will raise a SpecialFileError for unsupported file types - copy_function(srcname, dstname) - # catch the Error from the recursive copytree3 so that we can - # continue with other files + else: + # Will raise a SpecialFileError for unsupported file types + copy_function(srcname, dstname) + # catch the Error from the recursive copytree3 so that we can + # continue with other files except shutil.Error as err: - errors.extend(err.args[0]) + errors.extend(err.args[0]) except EnvironmentError as why: - errors.append((srcname, dstname, str(why))) - try: - shutil.copystat(src, dst) + errors.append((srcname, dstname, str(why))) + try: + shutil.copystat(src, dst) except OSError as why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.extend((src, dst, str(why))) - if errors: + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.extend((src, dst, str(why))) + if errors: raise shutil.Error(errors) diff --git a/library/python/func/__init__.py b/library/python/func/__init__.py index 7424361635..5e20802bc7 100644 --- a/library/python/func/__init__.py +++ b/library/python/func/__init__.py @@ -1,49 +1,49 @@ -import functools +import functools import threading import collections - - -def map0(func, value): - return func(value) if value is not None else value - - -def single(x): - if len(x) != 1: - raise Exception('Length of {} is not equal to 1'.format(x)) - return x[0] - - -class _Result(object): - pass - - -def lazy(func): - result = _Result() - + + +def map0(func, value): + return func(value) if value is not None else value + + +def single(x): + if len(x) != 1: + raise Exception('Length of {} is not equal to 1'.format(x)) + return x[0] + + +class _Result(object): + pass + + +def lazy(func): + result = _Result() + @functools.wraps(func) def wrapper(*args): - try: - return result.result - except AttributeError: + try: + return result.result + except AttributeError: result.result = func(*args) - - return result.result - - return wrapper - - -def lazy_property(fn): - attr_name = '_lazy_' + fn.__name__ - - @property - def _lazy_property(self): - if not hasattr(self, attr_name): - setattr(self, attr_name, fn(self)) - return getattr(self, attr_name) - - return _lazy_property - - + + return result.result + + return wrapper + + +def lazy_property(fn): + attr_name = '_lazy_' + fn.__name__ + + @property + def _lazy_property(self): + if not hasattr(self, attr_name): + setattr(self, attr_name, fn(self)) + return getattr(self, attr_name) + + return _lazy_property + + class classproperty(object): def __init__(self, func): self.func = func @@ -67,7 +67,7 @@ class lazy_classproperty(object): def memoize(limit=0, thread_local=False): assert limit >= 0 - def decorator(func): + def decorator(func): memory = {} lock = threading.Lock() @@ -86,7 +86,7 @@ def memoize(limit=0, thread_local=False): if len(keys) > limit: del memory[keys.popleft()] return memory[args] - + else: def get(args): @@ -112,15 +112,15 @@ def memoize(limit=0, thread_local=False): return wrapper - return decorator - - -# XXX: add test -def compose(*functions): - def compose2(f, g): - return lambda x: f(g(x)) + return decorator + + +# XXX: add test +def compose(*functions): + def compose2(f, g): + return lambda x: f(g(x)) - return functools.reduce(compose2, functions, lambda x: x) + return functools.reduce(compose2, functions, lambda x: x) class Singleton(type): diff --git a/library/python/func/ut/test_func.py b/library/python/func/ut/test_func.py index 3c4fad1a07..f3605bafcf 100644 --- a/library/python/func/ut/test_func.py +++ b/library/python/func/ut/test_func.py @@ -1,87 +1,87 @@ -import pytest +import pytest import threading - + import library.python.func as func - - -def test_map0(): + + +def test_map0(): assert None is func.map0(lambda x: x + 1, None) - assert 3 == func.map0(lambda x: x + 1, 2) + assert 3 == func.map0(lambda x: x + 1, 2) assert None is func.map0(len, None) - assert 2 == func.map0(len, [1, 2]) - - -def test_single(): - assert 1 == func.single([1]) - with pytest.raises(Exception): - assert 1 == func.single([]) - with pytest.raises(Exception): - assert 1 == func.single([1, 2]) - - -def test_memoize(): - class Counter(object): - @staticmethod - def inc(): - Counter._qty = getattr(Counter, '_qty', 0) + 1 - return Counter._qty - + assert 2 == func.map0(len, [1, 2]) + + +def test_single(): + assert 1 == func.single([1]) + with pytest.raises(Exception): + assert 1 == func.single([]) + with pytest.raises(Exception): + assert 1 == func.single([1, 2]) + + +def test_memoize(): + class Counter(object): + @staticmethod + def inc(): + Counter._qty = getattr(Counter, '_qty', 0) + 1 + return Counter._qty + @func.memoize() - def t1(a): - return a, Counter.inc() - + def t1(a): + return a, Counter.inc() + @func.memoize() - def t2(a): - return a, Counter.inc() - + def t2(a): + return a, Counter.inc() + @func.memoize() - def t3(a): - return a, Counter.inc() - + def t3(a): + return a, Counter.inc() + @func.memoize() - def t4(a): - return a, Counter.inc() - + def t4(a): + return a, Counter.inc() + @func.memoize() - def t5(a, b, c): - return a + b + c, Counter.inc() - + def t5(a, b, c): + return a + b + c, Counter.inc() + @func.memoize() - def t6(): - return Counter.inc() - + def t6(): + return Counter.inc() + @func.memoize(limit=2) def t7(a, _b): return a, Counter.inc() - assert (1, 1) == t1(1) - assert (1, 1) == t1(1) - assert (2, 2) == t1(2) - assert (2, 2) == t1(2) - - assert (1, 3) == t2(1) - assert (1, 3) == t2(1) - assert (2, 4) == t2(2) - assert (2, 4) == t2(2) - - assert (1, 5) == t3(1) - assert (1, 5) == t3(1) - assert (2, 6) == t3(2) - assert (2, 6) == t3(2) - - assert (1, 7) == t4(1) - assert (1, 7) == t4(1) - assert (2, 8) == t4(2) - assert (2, 8) == t4(2) - - assert (6, 9) == t5(1, 2, 3) - assert (6, 9) == t5(1, 2, 3) - assert (7, 10) == t5(1, 2, 4) - assert (7, 10) == t5(1, 2, 4) - - assert 11 == t6() - assert 11 == t6() - + assert (1, 1) == t1(1) + assert (1, 1) == t1(1) + assert (2, 2) == t1(2) + assert (2, 2) == t1(2) + + assert (1, 3) == t2(1) + assert (1, 3) == t2(1) + assert (2, 4) == t2(2) + assert (2, 4) == t2(2) + + assert (1, 5) == t3(1) + assert (1, 5) == t3(1) + assert (2, 6) == t3(2) + assert (2, 6) == t3(2) + + assert (1, 7) == t4(1) + assert (1, 7) == t4(1) + assert (2, 8) == t4(2) + assert (2, 8) == t4(2) + + assert (6, 9) == t5(1, 2, 3) + assert (6, 9) == t5(1, 2, 3) + assert (7, 10) == t5(1, 2, 4) + assert (7, 10) == t5(1, 2, 4) + + assert 11 == t6() + assert 11 == t6() + assert (1, 12) == t7(1, None) assert (2, 13) == t7(2, None) assert (1, 12) == t7(1, None) @@ -158,5 +158,5 @@ def test_memoize_thread_local(): th.join() -if __name__ == '__main__': - pytest.main([__file__]) +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/library/python/pytest/yatest_tools.py b/library/python/pytest/yatest_tools.py index 6b8b896394..72175cdffa 100644 --- a/library/python/pytest/yatest_tools.py +++ b/library/python/pytest/yatest_tools.py @@ -127,27 +127,27 @@ class Test(object): return [x.status for x in self.subtests].count(status) -class NoMd5FileException(Exception): - pass - - -TEST_SUBTEST_SEPARATOR = '::' - - -# TODO: extract color theme logic from ya -COLOR_THEME = { - 'test_name': 'light-blue', - 'test_project_path': 'dark-blue', - 'test_dir_desc': 'dark-magenta', - 'test_binary_path': 'light-gray', -} - - -# XXX: remove me -class YaCtx(object): - pass - -ya_ctx = YaCtx() +class NoMd5FileException(Exception): + pass + + +TEST_SUBTEST_SEPARATOR = '::' + + +# TODO: extract color theme logic from ya +COLOR_THEME = { + 'test_name': 'light-blue', + 'test_project_path': 'dark-blue', + 'test_dir_desc': 'dark-magenta', + 'test_binary_path': 'light-gray', +} + + +# XXX: remove me +class YaCtx(object): + pass + +ya_ctx = YaCtx() TRACE_FILE_NAME = "ytest.report.trace" diff --git a/library/python/ya.make b/library/python/ya.make index 2e1eb6e0e1..f534798383 100644 --- a/library/python/ya.make +++ b/library/python/ya.make @@ -138,7 +138,7 @@ RECURSE( path path/tests protobuf - pymain + pymain pyscopg2 pytest pytest-mongodb diff --git a/util/generic/fuzz/vector/ya.make b/util/generic/fuzz/vector/ya.make index b8614f6411..2f0e7931b5 100644 --- a/util/generic/fuzz/vector/ya.make +++ b/util/generic/fuzz/vector/ya.make @@ -1,4 +1,4 @@ -FUZZ() +FUZZ() OWNER( pg diff --git a/util/string/fuzzing/collapse/ya.make b/util/string/fuzzing/collapse/ya.make index b8614f6411..2f0e7931b5 100644 --- a/util/string/fuzzing/collapse/ya.make +++ b/util/string/fuzzing/collapse/ya.make @@ -1,4 +1,4 @@ -FUZZ() +FUZZ() OWNER( pg diff --git a/util/string/fuzzing/strtod/ya.make b/util/string/fuzzing/strtod/ya.make index b8614f6411..2f0e7931b5 100644 --- a/util/string/fuzzing/strtod/ya.make +++ b/util/string/fuzzing/strtod/ya.make @@ -1,4 +1,4 @@ -FUZZ() +FUZZ() OWNER( pg diff --git a/util/system/src_root.h b/util/system/src_root.h index 4f2d9f5ee6..702522d062 100644 --- a/util/system/src_root.h +++ b/util/system/src_root.h @@ -1,8 +1,8 @@ #pragma once -#include "compiler.h" +#include "compiler.h" #include "defaults.h" - + #include <type_traits> namespace NPrivate { @@ -48,10 +48,10 @@ namespace NPrivate { constexpr unsigned RootPrefixLength(const TStaticBuf& f) noexcept { if (IsProperPrefix(ArcRoot, f)) { return ArcRoot.Len + 1; - } + } if (IsProperPrefix(BuildRoot, f)) { return BuildRoot.Len + 1; - } + } return 0; } diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 70bb65514c..7eb158fa65 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -12,7 +12,7 @@ IF (OS_WINDOWS) NO_OPTIMIZE_PY_PROTOS() ENDIF() -SRCS( +SRCS( alloc.proto base.proto bind_channel_storage_pool.proto |